@Test public void testNull() { assertFunction("ngrams(ARRAY['foo', NULL, 'bar'], 2)", new ArrayType(new ArrayType(createVarcharType(3))), ImmutableList.of( asList("foo", null), asList(null, "bar"))); assertFunction("ngrams(ARRAY [NULL, NULL, NULL], 2)", new ArrayType(new ArrayType(UNKNOWN)), ImmutableList.of( asList(null, null), asList(null, null))); assertFunction("ngrams(ARRAY [NULL, 3, NULL], 2)", new ArrayType(new ArrayType(INTEGER)), ImmutableList.of( asList(null, 3), asList(3, null))); }
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder(); for (Field field : scope.getRelationType().getVisibleFields()) { Symbol symbol = symbolAllocator.newSymbol(field); ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder(); ImmutableList.Builder<Expression> values = ImmutableList.builder(); ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder(); Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator(); for (Expression expression : node.getExpressions()) { argumentSymbols.add(inputSymbol); if (type instanceof ArrayType) { Type elementType = ((ArrayType) type).getElementType(); if (!SystemSessionProperties.isLegacyUnnest(session) && elementType instanceof RowType) { ImmutableList.Builder<Symbol> unnestSymbolBuilder = ImmutableList.builder(); for (int i = 0; i < ((RowType) elementType).getFields().size(); i++) { unnestSymbolBuilder.add(unnestedSymbolsIterator.next()); Optional<Symbol> ordinalitySymbol = node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty(); checkState(!unnestedSymbolsIterator.hasNext(), "Not all output symbols were matched with input symbols"); ValuesNode valuesNode = new ValuesNode(idAllocator.getNextId(), argumentSymbols.build(), ImmutableList.of(values.build()));
private static Block createChannel(int positionCount, int arraySize, ArrayType arrayType) { BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, positionCount); for (int position = 0; position < positionCount; position++) { BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); for (int i = 0; i < arraySize; i++) { if (arrayType.getElementType().getJavaType() == long.class) { arrayType.getElementType().writeLong(entryBuilder, ThreadLocalRandom.current().nextLong()); } else { throw new UnsupportedOperationException(); } } blockBuilder.closeEntry(); } return blockBuilder.build(); }
@ScalarFunction(hidden = true) @SqlType("array(unknown)") public static Block arrayConstructor() { BlockBuilder blockBuilder = new ArrayType(UNKNOWN).createBlockBuilder(null, 0); return blockBuilder.build(); } }
@Override public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionRegistry functionRegistry) { Type keyType = boundVariables.getTypeVariable("K"); Type valueType = boundVariables.getTypeVariable("V"); Type outputType = typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of( TypeSignatureParameter.of(keyType.getTypeSignature()), TypeSignatureParameter.of(new ArrayType(valueType).getTypeSignature()))); return generateAggregation(keyType, valueType, outputType); }
@Override public Block[] getSequenceBlocks(int start, int length) { ArrayType arrayType = new ArrayType(BIGINT); BlockBuilder blockBuilder = arrayType.createBlockBuilder(null, length); for (int i = start; i < start + length; i++) { arrayType.writeObject(blockBuilder, arrayBlockOf(BIGINT, i)); } return new Block[] {blockBuilder.build()}; }
@Test public void testParquetTupleDomainStructArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array_struct", HiveType.valueOf("array<struct<a:int>>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); RowType.Field rowField = new RowType.Field(Optional.of("a"), INTEGER); RowType rowType = RowType.from(ImmutableList.of(rowField)); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(rowType)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array_struct", new GroupType(REPEATED, "bag", new GroupType(OPTIONAL, "array_element", new PrimitiveType(OPTIONAL, INT32, "a"))))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@Test(groups = "hive") public abstract class AbstractTestHiveFileFormats public static final List<TestColumn> TEST_COLUMNS = ImmutableList.<TestColumn>builder() .add(new TestColumn("p_empty_string", javaStringObjectInspector, "", Slices.EMPTY_SLICE, true)) .add(new TestColumn("p_string", javaStringObjectInspector, "test", Slices.utf8Slice("test"), true)) .add(new TestColumn("t_array_empty", getStandardListObjectInspector(javaStringObjectInspector), ImmutableList.of(), arrayBlockOf(createUnboundedVarcharType()))) .add(new TestColumn("t_array_string", getStandardListObjectInspector(javaStringObjectInspector), ImmutableList.of("test"), arrayBlockOf(createUnboundedVarcharType(), "test"))) mapBlockOf(createUnboundedVarcharType(), new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER))), "test", arrayBlockOf(RowType.anonymous(ImmutableList.of(INTEGER)), rowBlockOf(ImmutableList.of(INTEGER), 1L))))) .add(new TestColumn("t_map_null_key_complex_value", getStandardMapObjectInspector( mapBlockOf(new ArrayType(createUnboundedVarcharType()), mapType(BIGINT, BOOLEAN), arrayBlockOf(createUnboundedVarcharType(), "k", "ka"), mapBlockOf(BIGINT, BOOLEAN, 16L, false)))) .add(new TestColumn("t_struct_nested", getStandardStructObjectInspector(ImmutableList.of("struct_field"), ImmutableList.of(getStandardListObjectInspector(javaStringObjectInspector))), ImmutableList.of(ImmutableList.of("1", "2", "3")), rowBlockOf(ImmutableList.of(new ArrayType(createUnboundedVarcharType())), arrayBlockOf(createUnboundedVarcharType(), "1", "2", "3")))) RowType.anonymous(ImmutableList.of(INTEGER, createUnboundedVarcharType()))), session); HiveFileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory")); hiveFileWriter.appendRows(page); hiveFileWriter.commit();
private static PlanNode addPartitioningNodes(Context context, PlanNode node, Symbol partitionSymbol, KdbTree kdbTree, Expression geometry, Optional<Expression> radius) { Assignments.Builder projections = Assignments.builder(); for (Symbol outputSymbol : node.getOutputSymbols()) { projections.putIdentity(outputSymbol); } ImmutableList.Builder<Expression> partitioningArguments = ImmutableList.<Expression>builder() .add(new Cast(new StringLiteral(KdbTreeUtils.toJson(kdbTree)), KDB_TREE_TYPENAME)) .add(geometry); radius.map(partitioningArguments::add); FunctionCall partitioningFunction = new FunctionCall(QualifiedName.of("spatial_partitions"), partitioningArguments.build()); Symbol partitionsSymbol = context.getSymbolAllocator().newSymbol(partitioningFunction, new ArrayType(INTEGER)); projections.put(partitionsSymbol, partitioningFunction); return new UnnestNode( context.getIdAllocator().getNextId(), new ProjectNode(context.getIdAllocator().getNextId(), node, projections.build()), node.getOutputSymbols(), ImmutableMap.of(partitionsSymbol, ImmutableList.of(partitionSymbol)), Optional.empty()); }
@Test public void testParquetTupleDomainPrimitiveArray() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_array", HiveType.valueOf("array<int>"), parseTypeSignature(StandardTypes.ARRAY), 0, REGULAR, Optional.empty()); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(INTEGER)))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_array", new GroupType(REPEATED, "bag", new PrimitiveType(OPTIONAL, INT32, "array_element")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@BeforeClass public void setupClass() { AccumuloColumnHandle c1 = new AccumuloColumnHandle("id", Optional.empty(), Optional.empty(), VARCHAR, 0, "", false); AccumuloColumnHandle c2 = new AccumuloColumnHandle("age", Optional.of("cf"), Optional.of("age"), BIGINT, 1, "", true); AccumuloColumnHandle c3 = new AccumuloColumnHandle("firstname", Optional.of("cf"), Optional.of("firstname"), VARCHAR, 2, "", true); AccumuloColumnHandle c4 = new AccumuloColumnHandle("arr", Optional.of("cf"), Optional.of("arr"), new ArrayType(VARCHAR), 3, "", true); table = new AccumuloTable("default", "index_test_table", ImmutableList.of(c1, c2, c3, c4), "id", true, LexicoderRowSerializer.class.getCanonicalName(), null); m1 = new Mutation(M1_ROWID); m1.put(CF, AGE, AGE_VALUE); m1.put(CF, FIRSTNAME, M1_FNAME_VALUE); m1.put(CF, SENDERS, M1_ARR_VALUE); m2 = new Mutation(M2_ROWID); m2.put(CF, AGE, AGE_VALUE); m2.put(CF, FIRSTNAME, M2_FNAME_VALUE); m2.put(CF, SENDERS, M2_ARR_VALUE); ColumnVisibility visibility1 = new ColumnVisibility("private"); ColumnVisibility visibility2 = new ColumnVisibility("moreprivate"); m1v = new Mutation(M1_ROWID); m1v.put(CF, AGE, visibility1, AGE_VALUE); m1v.put(CF, FIRSTNAME, visibility1, M1_FNAME_VALUE); m1v.put(CF, SENDERS, visibility2, M1_ARR_VALUE); m2v = new Mutation(M2_ROWID); m2v.put(CF, AGE, visibility1, AGE_VALUE); m2v.put(CF, FIRSTNAME, visibility2, M2_FNAME_VALUE); m2v.put(CF, SENDERS, visibility2, M2_ARR_VALUE); }
getStandardListObjectInspector(javaStringObjectInspector), javaIntObjectInspector)); Type struct1Type = RowType.from(asList( field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("intField", INTEGER))); Type struct2Type = RowType.from(asList( field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("structField", struct1Type))); Type struct3Type = RowType.from(asList( field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("booleanField", BOOLEAN))); Type struct4Type = RowType.from(asList( field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("structField", struct3Type))); getStandardListObjectInspector(javaBooleanObjectInspector), getStandardMapObjectInspector(javaStringObjectInspector, javaStringObjectInspector)); List<Type> types = ImmutableList.of(struct1Type, struct2Type, struct3Type, struct4Type, mapType(INTEGER, DOUBLE), new ArrayType(BOOLEAN), mapType(VARCHAR, VARCHAR));
@Test public void testRowFromString() expected.addField(new Field(AccumuloRowSerializer.getBlockFromArray(VARCHAR, ImmutableList.of("a", "b", "c")), new ArrayType(VARCHAR))); expected.addField(true, BOOLEAN); expected.addField(new Field(new Date(new GregorianCalendar(1999, 0, 1).getTime().getTime()), DATE)); schema.addRowId("a", new ArrayType(VARCHAR)); schema.addColumn("b", Optional.of("b"), Optional.of("b"), BOOLEAN); schema.addColumn("c", Optional.of("c"), Optional.of("c"), DATE); schema.addColumn("d", Optional.of("d"), Optional.of("d"), DOUBLE); schema.addColumn("e", Optional.of("e"), Optional.of("e"), REAL);
@Setup public void setup() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); ExpressionCompiler compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0)); ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); Block[] blocks = new Block[TYPES.size()]; Type returnType = new ArrayType(BOOLEAN); for (int i = 0; i < TYPES.size(); i++) { Type elementType = TYPES.get(i); ArrayType arrayType = new ArrayType(elementType); Signature signature = new Signature("transform", FunctionKind.SCALAR, returnType.getTypeSignature(), arrayType.getTypeSignature(), parseTypeSignature("function(bigint,boolean)")); Signature greaterThan = new Signature("$operator$" + GREATER_THAN.name(), FunctionKind.SCALAR, BOOLEAN.getTypeSignature(), BIGINT.getTypeSignature(), BIGINT.getTypeSignature()); projectionsBuilder.add(new CallExpression(signature, returnType, ImmutableList.of( new InputReferenceExpression(0, arrayType), new LambdaDefinitionExpression( ImmutableList.of(BIGINT), ImmutableList.of("x"), new CallExpression(greaterThan, BOOLEAN, ImmutableList.of(new VariableReferenceExpression("x", BIGINT), new ConstantExpression(0L, BIGINT))))))); blocks[i] = createChannel(POSITIONS, ARRAY_SIZE, arrayType); } ImmutableList<RowExpression> projections = projectionsBuilder.build(); pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections).get(); pageBuilder = new PageBuilder(projections.stream().map(RowExpression::getType).collect(Collectors.toList())); page = new Page(blocks); }
switch (name) { case "fix-width": arrayType = new ArrayType(DOUBLE); elementsBlock = createFixWidthValueBlock(POSITIONS, arraySize); break; case "var-width": arrayType = new ArrayType(createUnboundedVarcharType()); elementsBlock = createVarWidthValueBlock(POSITIONS, arraySize); break; case "dictionary": arrayType = new ArrayType(createUnboundedVarcharType()); elementsBlock = createDictionaryValueBlock(POSITIONS, arraySize); break; case "array": arrayType = new ArrayType(new ArrayType(createUnboundedVarcharType())); elementsBlock = createArrayBlock(POSITIONS * arraySize, createVarWidthValueBlock(POSITIONS, arraySize)); break; ImmutableList.Builder<RowExpression> projectionsBuilder = ImmutableList.builder(); arrayType.getElementType().getTypeSignature(), arrayType.getTypeSignature(), BIGINT.getTypeSignature()); for (int i = 0; i < arraySize; i++) { projectionsBuilder.add(new CallExpression( signature, arrayType.getElementType(), ImmutableList.of(field(0, arrayType), constant((long) i + 1, BIGINT))));
@Override public ScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionRegistry functionRegistry) { Type leftElementType = boundVariables.getTypeVariable("T"); Type rightElementType = boundVariables.getTypeVariable("U"); Type outputElementType = boundVariables.getTypeVariable("R"); ArrayType outputArrayType = new ArrayType(outputElementType); return new ScalarFunctionImplementation( false, ImmutableList.of( valueTypeArgumentProperty(RETURN_NULL_ON_NULL), valueTypeArgumentProperty(RETURN_NULL_ON_NULL), functionTypeArgumentProperty(BinaryFunctionInterface.class)), METHOD_HANDLE.bindTo(leftElementType).bindTo(rightElementType).bindTo(outputArrayType), Optional.of(STATE_FACTORY.bindTo(outputArrayType)), isDeterministic()); }
@Test public void testCastWithJsonParse() assertCastWithJsonParse("[[1,1], [2,2]]", "ARRAY<ARRAY<INTEGER>>", new ArrayType(new ArrayType(INTEGER)), ImmutableList.of(ImmutableList.of(1, 1), ImmutableList.of(2, 2))); assertInvalidCastWithJsonParse("[1, \"abc\"]", "ARRAY<INTEGER>", "Cannot cast to array(integer). Cannot cast 'abc' to INT\n[1, \"abc\"]"); assertCastWithJsonParse("{\"a\"\n:1, \"b\":\t2}", "MAP<VARCHAR,INTEGER>", mapType(VARCHAR, INTEGER), ImmutableMap.of("a", 1, "b", 2)); assertInvalidCastWithJsonParse("{\"[1, 1]\":[2, 2]}", "MAP<ARRAY<INTEGER>,ARRAY<INTEGER>>", "Cannot cast JSON to map(array(integer),array(integer))"); assertInvalidCastWithJsonParse("{true: false, false:false}", "MAP<BOOLEAN,BOOLEAN>", "Cannot cast to map(boolean,boolean).\n{true: false, false:false}"); "{\"a\" \n :1, \"b\": \t [2, 3]}", "ROW(a INTEGER, b ARRAY<INTEGER>)", RowType.from(ImmutableList.of( RowType.field("a", INTEGER), RowType.field("b", new ArrayType(INTEGER)))), ImmutableList.of(1, ImmutableList.of(2, 3))); assertCastWithJsonParse( "[ 1, [2, 3] ]", "ROW(INTEGER, ARRAY<INTEGER>)", RowType.anonymous(ImmutableList.of(INTEGER, new ArrayType(INTEGER))), ImmutableList.of(1, ImmutableList.of(2, 3))); assertInvalidCastWithJsonParse(
@Override protected Scope visitUnnest(Unnest node, Optional<Scope> scope) ImmutableList.Builder<Field> outputFields = ImmutableList.builder(); for (Expression expression : node.getExpressions()) { ExpressionAnalysis expressionAnalysis = analyzeExpression(expression, createScope(scope)); Type expressionType = expressionAnalysis.getType(expression); if (expressionType instanceof ArrayType) { Type elementType = ((ArrayType) expressionType).getElementType(); if (!SystemSessionProperties.isLegacyUnnest(session) && elementType instanceof RowType) { ((RowType) elementType).getFields().stream() .map(field -> Field.newUnqualified(field.getName(), field.getType())) .forEach(outputFields::add); outputFields.add(Field.newUnqualified(Optional.empty(), elementType)); outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getKeyType())); outputFields.add(Field.newUnqualified(Optional.empty(), ((MapType) expressionType).getValueType()));
@Test public void testListBlock() { List<InnerStruct> array = new ArrayList<>(2); array.add(new InnerStruct(8, 9L)); array.add(new InnerStruct(10, 11L)); ListHolder listHolder = new ListHolder(); listHolder.array = array; com.facebook.presto.spi.type.Type rowType = RowType.anonymous(ImmutableList.of(INTEGER, BIGINT)); com.facebook.presto.spi.type.Type arrayOfRowType = RowType.anonymous(ImmutableList.of(new ArrayType(rowType))); Block actual = toBinaryBlock(arrayOfRowType, listHolder, getInspector(ListHolder.class)); BlockBuilder blockBuilder = rowType.createBlockBuilder(null, 1024); rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(INTEGER, BIGINT), 8, 9L)); rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(INTEGER, BIGINT), 10, 11L)); Block expected = rowBlockOf(ImmutableList.of(new ArrayType(rowType)), blockBuilder.build()); assertBlockEquals(actual, expected); }