public VarcharToIntegerNumberCoercer(HiveType type) { if (type.equals(HIVE_BYTE)) { minValue = Byte.MIN_VALUE; maxValue = Byte.MAX_VALUE; } else if (type.equals(HIVE_SHORT)) { minValue = Short.MIN_VALUE; maxValue = Short.MAX_VALUE; } else if (type.equals(HIVE_INT)) { minValue = Integer.MIN_VALUE; maxValue = Integer.MAX_VALUE; } else if (type.equals(HIVE_LONG)) { minValue = Long.MIN_VALUE; maxValue = Long.MAX_VALUE; } else { throw new PrestoException(NOT_SUPPORTED, format("Could not create Coercer from varchar to %s", type)); } }
@Override public boolean canCoerce(HiveType fromHiveType, HiveType toHiveType) { Type fromType = typeManager.getType(fromHiveType.getTypeSignature()); Type toType = typeManager.getType(toHiveType.getTypeSignature()); if (fromType instanceof VarcharType) { return toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG); } if (toType instanceof VarcharType) { return fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG); } if (fromHiveType.equals(HIVE_BYTE)) { return toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG); } if (fromHiveType.equals(HIVE_SHORT)) { return toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG); } if (fromHiveType.equals(HIVE_INT)) { return toHiveType.equals(HIVE_LONG); } if (fromHiveType.equals(HIVE_FLOAT)) { return toHiveType.equals(HIVE_DOUBLE); } return canCoerceForList(fromHiveType, toHiveType) || canCoerceForMap(fromHiveType, toHiveType) || canCoerceForStruct(fromHiveType, toHiveType); }
if (toType instanceof VarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { return new IntegerNumberToVarcharCoercer(fromType, toType); else if (fromType instanceof VarcharType && (toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) { return new VarcharToIntegerNumberCoercer(fromType, toType); else if (fromHiveType.equals(HIVE_BYTE) && toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(fromType, toType); else if (fromHiveType.equals(HIVE_SHORT) && toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(fromType, toType); else if (fromHiveType.equals(HIVE_INT) && toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(fromType, toType); else if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) { return new FloatToDoubleCoercer();
if (toType instanceof VarcharType && (fromHiveType.equals(HIVE_BYTE) || fromHiveType.equals(HIVE_SHORT) || fromHiveType.equals(HIVE_INT) || fromHiveType.equals(HIVE_LONG))) { return new IntegerNumberToVarcharCoercer(); else if (fromType instanceof VarcharType && (toHiveType.equals(HIVE_BYTE) || toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG))) { return new VarcharToIntegerNumberCoercer(toHiveType); else if (fromHiveType.equals(HIVE_BYTE) && toHiveType.equals(HIVE_SHORT) || toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(); else if (fromHiveType.equals(HIVE_SHORT) && toHiveType.equals(HIVE_INT) || toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(); else if (fromHiveType.equals(HIVE_INT) && toHiveType.equals(HIVE_LONG)) { return new IntegerNumberUpscaleCoercer(); else if (fromHiveType.equals(HIVE_FLOAT) && toHiveType.equals(HIVE_DOUBLE)) { return new FloatToDoubleCoercer();
public StructCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); List<HiveType> fromFieldTypes = extractStructFieldTypes(fromHiveType); List<HiveType> toFieldTypes = extractStructFieldTypes(toHiveType); this.coercers = new Function[toFieldTypes.size()]; this.nullBlocks = new Block[toFieldTypes.size()]; for (int i = 0; i < coercers.length; i++) { if (i >= fromFieldTypes.size()) { nullBlocks[i] = toFieldTypes.get(i).getType(typeManager).createBlockBuilder(null, 1).appendNull().build(); } else if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i))) { coercers[i] = createCoercer(typeManager, fromFieldTypes.get(i), toFieldTypes.get(i)); } } }
HiveType inputHiveType = inputColumnMap.get(columnName).getHiveType(); if (!fileColumnHiveType.equals(inputHiveType)) {
private boolean hasTheSameSchema(Table newTable, Table existingTable) { List<Column> newTableColumns = newTable.getDataColumns(); List<Column> existingTableColumns = existingTable.getDataColumns(); if (newTableColumns.size() != existingTableColumns.size()) { return false; } for (Column existingColumn : existingTableColumns) { if (newTableColumns.stream() .noneMatch(newColumn -> newColumn.getName().equals(existingColumn.getName()) && newColumn.getType().equals(existingColumn.getType()))) { return false; } } return true; }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType); }
private boolean canCoerceForStruct(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.STRUCT) || !toHiveType.getCategory().equals(Category.STRUCT)) { return false; } List<String> fromFieldNames = ((StructTypeInfo) fromHiveType.getTypeInfo()).getAllStructFieldNames(); List<String> toFieldNames = ((StructTypeInfo) toHiveType.getTypeInfo()).getAllStructFieldNames(); List<HiveType> fromFieldTypes = extractStructFieldTypes(fromHiveType); List<HiveType> toFieldTypes = extractStructFieldTypes(toHiveType); // Rule: // * Fields may be added or dropped from the end. // * For all other field indices, the corresponding fields must have // the same name, and the type must be coercible. for (int i = 0; i < min(fromFieldTypes.size(), toFieldTypes.size()); i++) { if (!fromFieldNames.get(i).equals(toFieldNames.get(i))) { return false; } if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i)) && !canCoerce(fromFieldTypes.get(i), toFieldTypes.get(i))) { return false; } } return true; } }
public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.fromElementType = fromElementHiveType.getType(typeManager); this.toType = toHiveType.getType(typeManager); this.toElementType = toElementHiveType.getType(typeManager); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType, bridgingRecordCursor); this.pageBuilder = elementCoercer == null ? null : new PageBuilder(ImmutableList.of(toType)); }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); }
HiveType tableType = tableColumns.get(i).getType(); HiveType partitionType = partitionColumns.get(i).getType(); if (!tableType.equals(partitionType)) { if (!coercionPolicy.canCoerce(partitionType, tableType)) { throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" +
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.fromKeyValueTypes = fromHiveType.getType(typeManager).getTypeParameters(); this.toType = toHiveType.getType(typeManager); this.toKeyValueTypes = toType.getTypeParameters(); this.coercers = new Coercer[2]; coercers[0] = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType, bridgingRecordCursor); coercers[1] = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType, bridgingRecordCursor); this.pageBuilder = coercers[0] == null && coercers[1] == null ? null : new PageBuilder(ImmutableList.of(toType)); }
HiveType tableType = tableColumns.get(i).getType(); HiveType partitionType = existingPartitionColumns.get(i).getType(); if (!tableType.equals(partitionType)) { throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + "You are trying to write into an existing partition in a table. " +
private boolean canCoerceForMap(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.MAP) || !toHiveType.getCategory().equals(Category.MAP)) { return false; } HiveType fromKeyType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); return (fromKeyType.equals(toKeyType) || canCoerce(fromKeyType, toKeyType)) && (fromValueType.equals(toValueType) || canCoerce(fromValueType, toValueType)); }
private boolean canCoerceForList(HiveType fromHiveType, HiveType toHiveType) { if (!fromHiveType.getCategory().equals(Category.LIST) || !toHiveType.getCategory().equals(Category.LIST)) { return false; } HiveType fromElementType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); return fromElementType.equals(toElementType) || canCoerce(fromElementType, toElementType); }
@Override public LazyBlockLoader<LazySliceArrayBlock> variableWidthBlockLoader(RcFileColumnsBatch batch, int fieldId, HiveType hiveType, ObjectInspector fieldInspector) { if (HIVE_STRING.equals(hiveType) || HIVE_BINARY.equals(hiveType)) { return new LazySliceBlockLoader(batch, fieldId); } throw new UnsupportedOperationException("Unsupported column type: " + hiveType); }
@Override public LazyBlockLoader<LazySliceArrayBlock> variableWidthBlockLoader(RcFileColumnsBatch batch, int fieldId, HiveType hiveType, ObjectInspector fieldInspector) { if (HIVE_STRING.equals(hiveType)) { return new LazyStringBlockLoader(batch, fieldId); } if (HIVE_BINARY.equals(hiveType)) { return new LazyBinaryBlockLoader(batch, fieldId); } throw new UnsupportedOperationException("Unsupported column type: " + hiveType); }
private void parseStringColumn(int column, byte[] bytes, int start, int length) { checkState(VALID_HIVE_STRING_TYPES.contains(hiveTypes[column]), "%s is not a valid STRING type", hiveTypes[column]); if (length == 0) { nulls[column] = true; } else { nulls[column] = false; // TODO: zero length BINARY is not supported. See https://issues.apache.org/jira/browse/HIVE-2483 if (hiveTypes[column].equals(HiveType.HIVE_STRING) && (length == 1) && bytes[start] == HIVE_EMPTY_STRING_BYTE) { slices[column] = Slices.EMPTY_SLICE; } else { slices[column] = Slices.wrappedBuffer(Arrays.copyOfRange(bytes, start, start + length)); } } }
private void parseStringColumn(int column, byte[] bytes, int start, int length) { boolean wasNull; if (length == "\\N".length() && bytes[start] == '\\' && bytes[start + 1] == 'N') { wasNull = true; } else { slices[column] = Slices.wrappedBuffer(Arrays.copyOfRange(bytes, start, start + length)); // this is unbelievably stupid but Hive base64 encodes binary data in a binary file format if (hiveTypes[column].equals(HiveType.HIVE_BINARY)) { // and yes we end up with an extra copy here because the Base64 only handles whole arrays slices[column] = base64Decode(slices[column].getBytes()); } wasNull = false; } nulls[column] = wasNull; }