public ListCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromElementHiveType = HiveType.valueOf(((ListTypeInfo) fromHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); HiveType toElementHiveType = HiveType.valueOf(((ListTypeInfo) toHiveType.getTypeInfo()).getListElementTypeInfo().getTypeName()); this.fromElementType = fromElementHiveType.getType(typeManager); this.toType = toHiveType.getType(typeManager); this.toElementType = toElementHiveType.getType(typeManager); this.elementCoercer = fromElementHiveType.equals(toElementHiveType) ? null : createCoercer(typeManager, fromElementHiveType, toElementHiveType, bridgingRecordCursor); this.pageBuilder = elementCoercer == null ? null : new PageBuilder(ImmutableList.of(toType)); }
public StructCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); List<HiveType> fromFieldHiveTypes = extractStructFieldTypes(fromHiveType); List<HiveType> toFieldHiveTypes = extractStructFieldTypes(toHiveType); this.fromFieldTypes = fromHiveType.getType(typeManager).getTypeParameters(); this.toType = toHiveType.getType(typeManager); this.toFieldTypes = toType.getTypeParameters(); this.coercers = new Coercer[toFieldHiveTypes.size()]; for (int i = 0; i < min(fromFieldHiveTypes.size(), toFieldHiveTypes.size()); i++) { if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i))) { coercers[i] = createCoercer(typeManager, fromFieldHiveTypes.get(i), toFieldHiveTypes.get(i), bridgingRecordCursor); } } this.pageBuilder = new PageBuilder(ImmutableList.of(toType)); }
public StructCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); List<HiveType> fromFieldTypes = extractStructFieldTypes(fromHiveType); List<HiveType> toFieldTypes = extractStructFieldTypes(toHiveType); this.coercers = new Function[toFieldTypes.size()]; this.nullBlocks = new Block[toFieldTypes.size()]; for (int i = 0; i < coercers.length; i++) { if (i >= fromFieldTypes.size()) { nullBlocks[i] = toFieldTypes.get(i).getType(typeManager).createBlockBuilder(null, 1).appendNull().build(); } else if (!fromFieldTypes.get(i).equals(toFieldTypes.get(i))) { coercers[i] = createCoercer(typeManager, fromFieldTypes.get(i), toFieldTypes.get(i)); } } }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); this.toType = requireNonNull(toHiveType, "toHiveType is null").getType(typeManager); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.keyCoercer = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType); this.valueCoercer = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType); }
private List<String> toConjuncts(List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> tupleDomain) { Builder<String> builder = builder(); for (HiveColumnHandle column : columns) { Type type = column.getHiveType().getType(typeManager); if (tupleDomain.getDomains().isPresent() && isSupported(type)) { Domain domain = tupleDomain.getDomains().get().get(column); if (domain != null) { builder.add(toPredicate(domain, type, column.getHiveColumnIndex())); } } } return builder.build(); }
public MapCoercer(TypeManager typeManager, HiveType fromHiveType, HiveType toHiveType, BridgingRecordCursor bridgingRecordCursor) { requireNonNull(typeManager, "typeManage is null"); requireNonNull(fromHiveType, "fromHiveType is null"); requireNonNull(toHiveType, "toHiveType is null"); this.bridgingRecordCursor = requireNonNull(bridgingRecordCursor, "bridgingRecordCursor is null"); HiveType fromKeyHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType fromValueHiveType = HiveType.valueOf(((MapTypeInfo) fromHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); HiveType toKeyHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapKeyTypeInfo().getTypeName()); HiveType toValueHiveType = HiveType.valueOf(((MapTypeInfo) toHiveType.getTypeInfo()).getMapValueTypeInfo().getTypeName()); this.fromKeyValueTypes = fromHiveType.getType(typeManager).getTypeParameters(); this.toType = toHiveType.getType(typeManager); this.toKeyValueTypes = toType.getTypeParameters(); this.coercers = new Coercer[2]; coercers[0] = fromKeyHiveType.equals(toKeyHiveType) ? null : createCoercer(typeManager, fromKeyHiveType, toKeyHiveType, bridgingRecordCursor); coercers[1] = fromValueHiveType.equals(toValueHiveType) ? null : createCoercer(typeManager, fromValueHiveType, toValueHiveType, bridgingRecordCursor); this.pageBuilder = coercers[0] == null && coercers[1] == null ? null : new PageBuilder(ImmutableList.of(toType)); }
.map(hiveType -> hiveType.getType(typeManager)) .collect(toList());
.map(hiveType -> hiveType.getType(typeManager)) .collect(toList());
.map(hiveType -> hiveType.getType(typeManager)) .collect(toList());
.map(column -> column.getHiveType().getType(typeManager)) .collect(toImmutableList());
ImmutableMap.Builder<Integer, Type> readColumns = ImmutableMap.builder(); for (HiveColumnHandle column : columns) { readColumns.put(column.getHiveColumnIndex(), column.getHiveType().getType(typeManager));
Type type = hiveType.getType(TYPE_MANAGER);
private static int computeActual(List<String> hiveTypeStrings, List<Object> hiveValues, int bucketCount, List<HiveType> hiveTypes, List<TypeInfo> hiveTypeInfos) { ImmutableList.Builder<Block> blockListBuilder = ImmutableList.builder(); Object[] nativeContainerValues = new Object[hiveValues.size()]; for (int i = 0; i < hiveTypeStrings.size(); i++) { Object hiveValue = hiveValues.get(i); Type type = hiveTypes.get(i).getType(TYPE_MANAGER); BlockBuilder blockBuilder = type.createBlockBuilder(null, 3); // prepend 2 nulls to make sure position is respected when HiveBucketing function blockBuilder.appendNull(); blockBuilder.appendNull(); appendToBlockBuilder(type, hiveValue, blockBuilder); Block block = blockBuilder.build(); blockListBuilder.add(block); nativeContainerValues[i] = toNativeContainerValue(type, hiveValue); } ImmutableList<Block> blockList = blockListBuilder.build(); int result1 = HiveBucketing.getHiveBucket(bucketCount, hiveTypeInfos, new Page(blockList.toArray(new Block[blockList.size()])), 2); int result2 = HiveBucketing.getHiveBucket(bucketCount, hiveTypeInfos, nativeContainerValues); assertEquals(result1, result2, "Overloads of getHiveBucket produced different result"); return result1; }
.collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager))); Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
.collect(toImmutableList()); Map<String, Type> columnTypes = handle.getInputColumns().stream() .collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager))); Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, partitionedBy, columnTypes);
TestColumn testColumn = testColumns.get(i); Type type = HiveType.valueOf(testColumn.getObjectInspector().getTypeName()).getType(TYPE_MANAGER); Object fieldFromCursor = getFieldFromCursor(cursor, type, i); if (fieldFromCursor == null) {
@Test public void testDecimalColumns() { TypeManager typeManager = new TypeRegistry(); IonSqlQueryBuilder queryBuilder = new IonSqlQueryBuilder(typeManager); List<HiveColumnHandle> columns = ImmutableList.of( new HiveColumnHandle("quantity", HiveType.valueOf("decimal(20,0)"), parseTypeSignature(DECIMAL), 0, REGULAR, Optional.empty()), new HiveColumnHandle("extendedprice", HiveType.valueOf("decimal(20,2)"), parseTypeSignature(DECIMAL), 1, REGULAR, Optional.empty()), new HiveColumnHandle("discount", HiveType.valueOf("decimal(10,2)"), parseTypeSignature(DECIMAL), 2, REGULAR, Optional.empty())); DecimalType decimalType = DecimalType.createDecimalType(10, 2); TupleDomain<HiveColumnHandle> tupleDomain = withColumnDomains( ImmutableMap.of( columns.get(0), Domain.create(ofRanges(Range.lessThan(DecimalType.createDecimalType(20, 0), longDecimal("50"))), false), columns.get(1), Domain.create(ofRanges(Range.equal(HiveType.valueOf("decimal(20,2)").getType(typeManager), longDecimal("0.05"))), false), columns.get(2), Domain.create(ofRanges(Range.range(decimalType, shortDecimal("0.0"), true, shortDecimal("0.02"), true)), false))); assertEquals("SELECT s._1, s._2, s._3 FROM S3Object s WHERE ((case s._1 when '' then null else CAST(s._1 AS DECIMAL(20,0)) end < 50)) AND " + "(case s._2 when '' then null else CAST(s._2 AS DECIMAL(20,2)) end = 0.05) AND ((case s._3 when '' then null else CAST(s._3 AS DECIMAL(10,2)) " + "end >= 0.00 AND case s._3 when '' then null else CAST(s._3 AS DECIMAL(10,2)) end <= 0.02))", queryBuilder.buildSql(columns, tupleDomain)); }
.map(LineItemColumn::getType) .map(TestHivePageSink::getHiveType) .map(hiveType -> hiveType.getType(TYPE_MANAGER)) .collect(toList());
.map(TestColumn::getType) .map(HiveType::valueOf) .map(type -> type.getType(TYPE_MANAGER)) .collect(toList());
private static int computeActual(List<String> hiveTypeStrings, List<Object> javaValues, int bucketCount, List<HiveType> hiveTypes, List<TypeInfo> hiveTypeInfos) { ImmutableList.Builder<Block> blockListBuilder = ImmutableList.builder(); for (int i = 0; i < hiveTypeStrings.size(); i++) { Object javaValue = javaValues.get(i); Type type = hiveTypes.get(i).getType(typeRegistry); BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 3); // prepend 2 nulls to make sure position is respected when HiveBucketing function blockBuilder.appendNull(); blockBuilder.appendNull(); appendToBlockBuilder(type, javaValue, blockBuilder); Block block = blockBuilder.build(); blockListBuilder.add(block); } ImmutableList<Block> blockList = blockListBuilder.build(); return HiveBucketing.getHiveBucket(hiveTypeInfos, blockList, 2, bucketCount); }