@Test public void testSchemaWithOptionalOptionalRequiredFields() throws Exception { MessageType parquetSchema = parseMessageType("message hive_schema {" + " optional group a {" + " optional group b {" + " optional group c {" + " required binary d (UTF8);" + " }" + " }" + " }" + "} "); Type cType = RowType.from(singletonList(field("d", VARCHAR))); Type bType = RowType.from(singletonList(field("c", cType))); Type aType = RowType.from(singletonList(field("b", bType))); Iterable<String> dValues = asList("d1", "d2", "d3", "d4", "d5", "d6", "d7"); Iterable<List> cValues = createNullableTestStructs(dValues); Iterable<List> bValues = createNullableTestStructs(cValues); List<List> aValues = createTestStructs(bValues); ObjectInspector cInspector = getStandardStructObjectInspector(singletonList("d"), singletonList(javaStringObjectInspector)); ObjectInspector bInspector = getStandardStructObjectInspector(singletonList("c"), singletonList(cInspector)); ObjectInspector aInspector = getStandardStructObjectInspector(singletonList("b"), singletonList(bInspector)); tester.testRoundTrip(aInspector, aValues, aValues, "a", aType, Optional.of(parquetSchema)); }
@Test public void testDifferentLength() { assertFunction( "zip_with(ARRAY[1], ARRAY['a', 'bc'], (x, y) -> (y, x))", new ArrayType(RowType.anonymous(ImmutableList.of(createVarcharType(2), INTEGER))), ImmutableList.of(ImmutableList.of("a", 1), asList("bc", null))); assertFunction( "zip_with(ARRAY[NULL, 2], ARRAY['a'], (x, y) -> (y, x))", new ArrayType(RowType.anonymous(ImmutableList.of(createVarcharType(1), INTEGER))), ImmutableList.of(asList("a", null), asList(null, 2))); assertFunction( "zip_with(ARRAY[NULL, NULL], ARRAY[NULL, 2, 1], (x, y) -> x + y)", new ArrayType(INTEGER), asList(null, null, null)); }
public static Block rowBlockOf(List<Type> parameterTypes, Object... values) { RowType rowType = RowType.anonymous(parameterTypes); BlockBuilder blockBuilder = rowType.createBlockBuilder(null, 1); BlockBuilder singleRowBlockWriter = blockBuilder.beginBlockEntry(); for (int i = 0; i < values.length; i++) { appendToBlockBuilder(parameterTypes.get(i), values[i], singleRowBlockWriter); } blockBuilder.closeEntry(); return rowType.getObject(blockBuilder, 0); }
@UsedByGeneratedCode public static Block zip(List<Type> types, Block... arrays) { int biggestCardinality = 0; for (Block array : arrays) { biggestCardinality = Math.max(biggestCardinality, array.getPositionCount()); } RowType rowType = RowType.anonymous(types); BlockBuilder outputBuilder = rowType.createBlockBuilder(null, biggestCardinality); for (int outputPosition = 0; outputPosition < biggestCardinality; outputPosition++) { BlockBuilder rowBuilder = outputBuilder.beginBlockEntry(); for (int fieldIndex = 0; fieldIndex < arrays.length; fieldIndex++) { if (arrays[fieldIndex].getPositionCount() <= outputPosition) { rowBuilder.appendNull(); } else { types.get(fieldIndex).appendTo(arrays[fieldIndex], outputPosition, rowBuilder); } } outputBuilder.closeEntry(); } return outputBuilder.build(); } }
@Test public void testParquetTupleDomainStruct() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_struct", HiveType.valueOf("struct<a:int,b:int>"), parseTypeSignature(StandardTypes.ROW), 0, REGULAR, Optional.empty()); RowType.Field rowField = new RowType.Field(Optional.of("my_struct"), INTEGER); RowType rowType = RowType.from(ImmutableList.of(rowField)); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(rowType))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_struct", new PrimitiveType(OPTIONAL, INT32, "a"), new PrimitiveType(OPTIONAL, INT32, "b"))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder(); for (Field field : scope.getRelationType().getVisibleFields()) { Symbol symbol = symbolAllocator.newSymbol(field); ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder(); ImmutableList.Builder<Expression> values = ImmutableList.builder(); ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder(); Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator(); argumentSymbols.add(inputSymbol); if (type instanceof ArrayType) { Type elementType = ((ArrayType) type).getElementType(); if (!SystemSessionProperties.isLegacyUnnest(session) && elementType instanceof RowType) { ImmutableList.Builder<Symbol> unnestSymbolBuilder = ImmutableList.builder(); for (int i = 0; i < ((RowType) elementType).getFields().size(); i++) { unnestSymbolBuilder.add(unnestedSymbolsIterator.next()); Optional<Symbol> ordinalitySymbol = node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty(); checkState(!unnestedSymbolsIterator.hasNext(), "Not all output symbols were matched with input symbols"); ValuesNode valuesNode = new ValuesNode(idAllocator.getNextId(), argumentSymbols.build(), ImmutableList.of(values.build()));
Scope scope = context.getContext().getScope(); Optional<ResolvedField> resolvedField = scope.tryResolveField(node, qualifiedName); if (resolvedField.isPresent()) { return handleResolvedField(node, resolvedField.get(), context); for (RowType.Field rowField : rowType.getFields()) { if (fieldName.equalsIgnoreCase(rowField.getName().orElse(null))) { rowFieldType = rowField.getType(); break; OptionalInt rowIndex = parseAnonymousRowFieldOrdinalAccess(fieldName, rowType.getFields()); if (rowIndex.isPresent()) { rowFieldType = rowType.getFields().get(rowIndex.getAsInt()).getType();
@Override public void setField(String name, StdData value) { BlockBuilder mutable = _rowType.createBlockBuilder(new PageBuilderStatus().createBlockBuilderStatus(), 1); BlockBuilder rowBlockBuilder = mutable.beginBlockEntry(); int i = 0; for (RowType.Field field : _rowType.getFields()) { if (field.getName().isPresent() && name.equals(field.getName().get())) { ((PrestoData) value).writeToBlock(rowBlockBuilder); } else { if (_block == null) { rowBlockBuilder.appendNull(); } else { field.getType().appendTo(_block, i, rowBlockBuilder); } } i++; } mutable.closeEntry(); _block = _rowType.getObject(mutable.build(), 0); }
@Override protected RowExpression visitDereferenceExpression(DereferenceExpression node, Void context) { RowType rowType = (RowType) getType(node.getBase()); String fieldName = node.getField().getValue(); List<Field> fields = rowType.getFields(); int index = -1; for (int i = 0; i < fields.size(); i++) { Field field = fields.get(i); if (field.getName().isPresent() && field.getName().get().equalsIgnoreCase(fieldName)) { checkArgument(index < 0, "Ambiguous field %s in type %s", field, rowType.getDisplayName()); index = i; } } if (legacyRowFieldOrdinalAccess && index < 0) { OptionalInt rowIndex = parseAnonymousRowFieldOrdinalAccess(fieldName, fields); if (rowIndex.isPresent()) { index = rowIndex.getAsInt(); } } checkState(index >= 0, "could not find field name: %s", node.getField()); Type returnType = getType(node); return call(dereferenceSignature(returnType, rowType), returnType, process(node.getBase(), context), constant(index, INTEGER)); }
private Block[] getBlocksForType(Type type, int positionCount) throws IOException { RowType rowType = (RowType) type; Block[] blocks = new Block[rowType.getFields().size()]; for (int i = 0; i < rowType.getFields().size(); i++) { Optional<String> fieldName = rowType.getFields().get(i).getName(); Type fieldType = rowType.getFields().get(i).getType(); if (!fieldName.isPresent()) { throw new IllegalArgumentException("Missing struct field name in type " + rowType); } String lowerCaseFieldName = fieldName.get().toLowerCase(Locale.ENGLISH); StreamReader streamReader = structFields.get(lowerCaseFieldName); if (streamReader != null) { streamReader.prepareNextRead(positionCount); blocks[i] = streamReader.readBlock(fieldType); } else { blocks[i] = getNullBlock(fieldType, positionCount); } } return blocks; }
@Test(groups = "hive") public abstract class AbstractTestHiveFileFormats public static final List<TestColumn> TEST_COLUMNS = ImmutableList.<TestColumn>builder() .add(new TestColumn("p_empty_string", javaStringObjectInspector, "", Slices.EMPTY_SLICE, true)) .add(new TestColumn("p_string", javaStringObjectInspector, "test", Slices.utf8Slice("test"), true)) mapBlockOf(createUnboundedVarcharType(), new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER))), "test", arrayBlockOf(RowType.anonymous(ImmutableList.of(INTEGER)), rowBlockOf(ImmutableList.of(INTEGER), 1L))))) .add(new TestColumn("t_map_null_key_complex_value", getStandardMapObjectInspector( mapBlockOf(new ArrayType(createUnboundedVarcharType()), mapType(BIGINT, BOOLEAN), arrayBlockOf(createUnboundedVarcharType(), "k", "ka"), mapBlockOf(BIGINT, BOOLEAN, 16L, false)))) INTEGER, createUnboundedVarcharType(), RowType.anonymous(ImmutableList.of(INTEGER, createUnboundedVarcharType()))), session); HiveFileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory")); hiveFileWriter.appendRows(page); hiveFileWriter.commit();
@Test(expectedExceptions = RuntimeException.class) public void testStructCreateByIndexFunctionError1() { assertFunction("struct_create_by_index(null, null)", RowType.from( ImmutableList.of(new RowType.Field(Optional.of("field0"), UNKNOWN), new RowType.Field(Optional.of("field1"), UNKNOWN))), null); } }
@Test public void testDoubleRowMap() { RowType innerRowType = RowType.from(ImmutableList.of( RowType.field("f1", BIGINT), RowType.field("f2", DOUBLE))); testMultimapAgg(DOUBLE, ImmutableList.of(1.0, 2.0, 3.0), innerRowType, ImmutableList.of(ImmutableList.of(1L, 1.0), ImmutableList.of(2L, 2.0), ImmutableList.of(3L, 3.0))); }
@Setup public void setup() { Signature signature = new Signature("$operator$CAST", SCALAR, RowType.anonymous(fromFieldTypes).getTypeSignature(), RowType.anonymous(toFieldTypes).getTypeSignature()); List<RowExpression> projections = ImmutableList.of( new CallExpression(signature, RowType.anonymous(fromFieldTypes), ImmutableList.of(field(0, RowType.anonymous(toFieldTypes))))); MetadataManager metadata = MetadataManager.createTestMetadataManager(); pageProcessor = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)) .compilePageProcessor(Optional.empty(), projections) .get(); Block[] fieldBlocks = fromFieldTypes.stream() .map(type -> createBlock(POSITION_COUNT, type)) .toArray(Block[]::new); Block rowBlock = fromFieldBlocks(POSITION_COUNT, Optional.empty(), fieldBlocks); page = new Page(rowBlock); }
private TypeCompatibility typeCompatibilityForRow(RowType firstType, RowType secondType) { List<Field> firstFields = firstType.getFields(); List<Field> secondFields = secondType.getFields(); if (firstFields.size() != secondFields.size()) { return TypeCompatibility.incompatible(); } ImmutableList.Builder<RowType.Field> fields = ImmutableList.builder(); boolean coercible = true; for (int i = 0; i < firstFields.size(); i++) { Type firstFieldType = firstFields.get(i).getType(); Type secondFieldType = secondFields.get(i).getType(); TypeCompatibility typeCompatibility = compatibility(firstFieldType, secondFieldType); if (!typeCompatibility.isCompatible()) { return TypeCompatibility.incompatible(); } Type commonParameterType = typeCompatibility.getCommonSuperType(); Optional<String> firstParameterName = firstFields.get(i).getName(); Optional<String> secondParameterName = secondFields.get(i).getName(); Optional<String> commonName = firstParameterName.equals(secondParameterName) ? firstParameterName : Optional.empty(); // ignore parameter name for coercible coercible &= typeCompatibility.isCoercible(); fields.add(new RowType.Field(commonName, commonParameterType)); } return TypeCompatibility.compatible(RowType.from(fields.build()), coercible); }
if (field.contains(".")) { String prefix = field.substring(0, field.lastIndexOf('.')); ImmutableList.Builder<Field> fieldsBuilder = ImmutableList.builder(); int size = field.split("\\.").length; Iterator<String> iterator = fieldsMap.navigableKeySet().iterator(); String name = iterator.next(); if (name.split("\\.").length == size && name.startsWith(prefix)) { Optional<String> columnName = Optional.of(name.substring(name.lastIndexOf('.') + 1)); Type columnType = fieldsMap.get(name); Field column = new Field(columnName, columnType); fieldsMap.put(prefix, RowType.from(fieldsBuilder.build()));
@Override protected Scope visitUnnest(Unnest node, Optional<Scope> scope) ImmutableList.Builder<Field> outputFields = ImmutableList.builder(); for (Expression expression : node.getExpressions()) { ExpressionAnalysis expressionAnalysis = analyzeExpression(expression, createScope(scope)); Type expressionType = expressionAnalysis.getType(expression); if (expressionType instanceof ArrayType) { Type elementType = ((ArrayType) expressionType).getElementType(); if (!SystemSessionProperties.isLegacyUnnest(session) && elementType instanceof RowType) { ((RowType) elementType).getFields().stream() .map(field -> Field.newUnqualified(field.getName(), field.getType())) .forEach(outputFields::add); outputFields.add(Field.newUnqualified(Optional.empty(), elementType)); outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getKeyType())); outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getValueType()));
@Test public void testStructBlock() com.facebook.presto.spi.type.Type rowType = RowType.anonymous(ImmutableList.of(INTEGER, BIGINT)); Block actual = toBinaryBlock(rowType, innerStruct, getInspector(InnerStruct.class)); Block expected = rowBlockOf(ImmutableList.of(INTEGER, BIGINT), 13, 14L); assertBlockEquals(actual, expected); outerStruct.innerStruct = new InnerStruct(18, 19L); com.facebook.presto.spi.type.Type innerRowType = RowType.anonymous(ImmutableList.of(INTEGER, BIGINT)); com.facebook.presto.spi.type.Type arrayOfInnerRowType = new ArrayType(innerRowType); com.facebook.presto.spi.type.Type mapOfInnerRowType = mapType(createUnboundedVarcharType(), innerRowType); List<com.facebook.presto.spi.type.Type> outerRowParameterTypes = ImmutableList.of(TINYINT, SMALLINT, INTEGER, BIGINT, REAL, DOUBLE, createUnboundedVarcharType(), createUnboundedVarcharType(), arrayOfInnerRowType, mapOfInnerRowType, innerRowType); com.facebook.presto.spi.type.Type outerRowType = RowType.anonymous(outerRowParameterTypes);