public HistogramStateSerializer(Type type) { this.type = type; this.serializedType = new MapType(type, BigintType.BIGINT); }
public KeyValuePairStateSerializer(Type keyType, Type valueType, boolean isMultiValue) { this.mapType = new MapType(keyType, valueType); this.isMultiValue = isMultiValue; }
private Type toPrestoMapType(final LogicalSchema.LogicalFieldSchema field) { final Type key = StringType.STRING; Type value = UnknownType.UNKNOWN; if (field.schema != null) { final List<LogicalSchema.LogicalFieldSchema> fields = field.schema.getFields(); if (fields.size() > 0) { value = toPrestoType(fields.get(0)); } } return new MapType(key, value); } }
@Override public Type createType(List<TypeParameter> parameters) { checkArgument(parameters.size() == 2, "Expected two parameters, got %s", parameters); TypeParameter firstParameter = parameters.get(0); TypeParameter secondParameter = parameters.get(1); checkArgument( firstParameter.getKind() == ParameterKind.TYPE && secondParameter.getKind() == ParameterKind.TYPE, "Expected key and type to be types, got %s", parameters); return new MapType(firstParameter.getType(), secondParameter.getType()); } }
@ScalarFunction("test_row") @SqlType("row<boolean,array(bigint),map(bigint,double)>('col0','col1','col2')") public static Block testRowBooleanArrayMap(@Nullable @SqlType(StandardTypes.BOOLEAN) Boolean arg1, @Nullable @SqlType("array(bigint)") Block arg2, @Nullable @SqlType("map(bigint,double)") Block arg3) { List<Type> parameterTypes = ImmutableList.of(BOOLEAN, new ArrayType(BIGINT), new MapType(BIGINT, DOUBLE)); return toStackRepresentation(parameterTypes, arg1, arg2, arg3); }
private static Map<Double, Double> extractSingleValue(Block block) throws IOException { MapType mapType = new MapType(DOUBLE, DOUBLE); return (Map<Double, Double>) mapType.getObjectValue(null, block, 0); }
@Test public void testDoubleMapMultimap() throws Exception { Type mapType = new MapType(VARCHAR, BIGINT); List<Double> expectedKeys = ImmutableList.of(1.0, 2.0, 3.0); List<Map<String, Long>> expectedValues = ImmutableList.of(ImmutableMap.of("a", 1L), ImmutableMap.of("b", 2L, "c", 3L, "d", 4L), ImmutableMap.of("a", 1L)); testMultimapAgg(DOUBLE, expectedKeys, mapType, expectedValues); }
@Test public void testRowDisplayName() { List<Type> types = asList(BOOLEAN, DOUBLE, new ArrayType(VARCHAR), new MapType(BOOLEAN, DOUBLE)); Optional<List<String>> names = Optional.of(asList("bool_col", "double_col", "array_col", "map_col")); RowType row = new RowType(types, names); assertEquals( row.getDisplayName(), "row(bool_col boolean, double_col double, array_col array(varchar), map_col map(boolean, double))"); }
@Test public void testRowDisplayNoColumnNames() { List<Type> types = asList(BOOLEAN, DOUBLE, new ArrayType(VARCHAR), new MapType(BOOLEAN, DOUBLE)); RowType row = new RowType(types, Optional.empty()); assertEquals( row.getDisplayName(), "row(boolean, double, array(varchar), map(boolean, double))"); } }
@Test public void testArrayHistograms() throws Exception { ArrayType arrayType = new ArrayType(VARCHAR); MapType mapType = new MapType(arrayType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), arrayType.getTypeSignature().toString())); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableList.of("a", "b", "c"), 1L, ImmutableList.of("d", "e", "f"), 1L, ImmutableList.of("c", "b", "a"), 1L), createStringArraysBlock(ImmutableList.of(ImmutableList.of("a", "b", "c"), ImmutableList.of("d", "e", "f"), ImmutableList.of("c", "b", "a")))); }
@Test public void testReuse() throws Exception { BytesWritable value = new BytesWritable(); byte[] first = "hello world".getBytes(UTF_8); value.set(first, 0, first.length); byte[] second = "bye".getBytes(UTF_8); value.set(second, 0, second.length); Type type = new TypeToken<Map<BytesWritable, Integer>>() {}.getType(); ObjectInspector inspector = getInspector(type); Block actual = getBlockObject(new MapType(VARCHAR, BIGINT), ImmutableMap.of(value, 0), inspector); Block expected = mapBlockOf(VARCHAR, BIGINT, "bye", 0); assertBlockEquals(actual, expected); }
@Test public void testMapBlock() { MapHolder holder = new MapHolder(); holder.map = new TreeMap<>(); holder.map.put("twelve", new InnerStruct(13, 14L)); holder.map.put("fifteen", new InnerStruct(16, 17L)); com.facebook.presto.spi.type.Type rowType = new RowType(ImmutableList.of(BIGINT, BIGINT), Optional.empty()); com.facebook.presto.spi.type.Type mapOfVarcharRowType = new RowType(ImmutableList.of(new MapType(VARCHAR, rowType)), Optional.empty()); Block actual = toBinaryBlock(mapOfVarcharRowType, holder, getInspector(MapHolder.class)); BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(VARCHAR, rowType), new BlockBuilderStatus(), 1024); VARCHAR.writeString(blockBuilder, "fifteen"); rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(BIGINT, BIGINT), 16, 17L)); VARCHAR.writeString(blockBuilder, "twelve"); rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(BIGINT, BIGINT), 13, 14L)); Block expected = rowBlockOf(ImmutableList.of(new MapType(VARCHAR, rowType)), blockBuilder); assertBlockEquals(actual, expected); }
@Test public void testDoubleArrayMap() throws Exception { ArrayType arrayType = new ArrayType(VARCHAR); MapType mapType = new MapType(DOUBLE, arrayType); InternalAggregationFunction aggFunc = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.DOUBLE, arrayType.getTypeSignature().toString())); assertAggregation( aggFunc, 1.0, ImmutableMap.of(1.0, ImmutableList.of("a", "b"), 2.0, ImmutableList.of("c", "d"), 3.0, ImmutableList.of("e", "f")), createDoublesBlock(1.0, 2.0, 3.0), createStringArraysBlock(ImmutableList.of(ImmutableList.of("a", "b"), ImmutableList.of("c", "d"), ImmutableList.of("e", "f")))); }
private static InternalAggregationFunction generateAggregation(Type keyType, Type valueType) { DynamicClassLoader classLoader = new DynamicClassLoader(MapAggregationFunction.class.getClassLoader()); List<Type> inputTypes = ImmutableList.of(keyType, valueType); Type outputType = new MapType(keyType, valueType); KeyValuePairStateSerializer stateSerializer = new KeyValuePairStateSerializer(keyType, valueType, false); Type intermediateType = stateSerializer.getSerializedType(); AggregationMetadata metadata = new AggregationMetadata( generateAggregationName(NAME, outputType, inputTypes), createInputParameterMetadata(keyType, valueType), INPUT_FUNCTION.bindTo(keyType).bindTo(valueType), null, null, COMBINE_FUNCTION, OUTPUT_FUNCTION, KeyValuePairsState.class, stateSerializer, new KeyValuePairsStateFactory(keyType, valueType), outputType, false); GenericAccumulatorFactoryBinder factory = new AccumulatorCompiler().generateAccumulatorFactoryBinder(metadata, classLoader); return new InternalAggregationFunction(NAME, inputTypes, intermediateType, outputType, true, false, factory); }
@Test public void testLargerHistograms() throws Exception { MapType mapType = new MapType(VARCHAR, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.VARCHAR)); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of("a", 25L, "b", 10L, "c", 12L, "d", 1L, "e", 2L), createStringsBlock("a", "b", "c", "d", "e", "e", "c", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c")); } }
@Test public void testMapHistograms() throws Exception { MapType innerMapType = new MapType(VARCHAR, VARCHAR); MapType mapType = new MapType(innerMapType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), innerMapType.getTypeSignature().toString())); BlockBuilder builder = innerMapType.createBlockBuilder(new BlockBuilderStatus(), 3); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("a", "b"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("c", "d"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("e", "f"))); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableMap.of("a", "b"), 1L, ImmutableMap.of("c", "d"), 1L, ImmutableMap.of("e", "f"), 1L), builder.build()); }
@Test public void testRowHistograms() throws Exception { RowType innerRowType = new RowType(ImmutableList.of(BIGINT, DOUBLE), Optional.of(ImmutableList.of("f1", "f2"))); MapType mapType = new MapType(innerRowType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), innerRowType.getTypeSignature().toString())); BlockBuilder builder = innerRowType.createBlockBuilder(new BlockBuilderStatus(), 3); innerRowType.writeObject(builder, testRowBigintBigint(1L, 1.0)); innerRowType.writeObject(builder, testRowBigintBigint(2L, 2.0)); innerRowType.writeObject(builder, testRowBigintBigint(3L, 3.0)); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableList.of(1L, 1.0), 1L, ImmutableList.of(2L, 2.0), 1L, ImmutableList.of(3L, 3.0), 1L), builder.build()); }
@Test public void testWithNulls() throws Exception { MapType mapType = new MapType(BIGINT, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.BIGINT)); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(1L, 1L, 2L, 1L), createLongsBlock(2L, null, 1L)); mapType = new MapType(BIGINT, BIGINT); aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.BIGINT)); assertAggregation( aggregationFunction, 1.0, null, createLongsBlock((Long) null)); }
@Test public void testFieldAccessor() throws Exception { // test_row has both (bigint, double) and (bigint, bigint) so this method is non-deterministic // assertFunction("test_row(1, NULL).col1", BIGINT, null); // test_row has both (boolean, boolean) and (boolean, array<bigint>), so this method is non-deterministic // assertFunction("test_row(TRUE, NULL).col1", BOOLEAN, null); assertFunction("test_row(1, CAST(NULL AS DOUBLE)).col1", DOUBLE, null); assertFunction("test_row(TRUE, CAST(NULL AS BOOLEAN)).col1", BOOLEAN, null); assertFunction("test_row(TRUE, CAST(NULL AS ARRAY<BIGINT>)).col1", new ArrayType(BIGINT), null); assertFunction("test_row(1.0, CAST(NULL AS VARCHAR)).col1", VARCHAR, null); assertFunction("test_row(1, 2).col0", BIGINT, 1); assertFunction("test_row(1, 'kittens').col1", VARCHAR, "kittens"); assertFunction("test_row(1, 2).\"col1\"", BIGINT, 2); assertFunction("array[test_row(1, 2)][1].col1", BIGINT, 2); assertFunction("test_row(FALSE, ARRAY [1, 2], MAP(ARRAY[1, 3], ARRAY[2.0, 4.0])).col1", new ArrayType(BIGINT), ImmutableList.of(1L, 2L)); assertFunction("test_row(FALSE, ARRAY [1, 2], MAP(ARRAY[1, 3], ARRAY[2.0, 4.0])).col2", new MapType(BIGINT, DOUBLE), ImmutableMap.of(1L, 2.0, 3L, 4.0)); assertFunction("test_row(1.0, ARRAY[test_row(31, 4.1), test_row(32, 4.2)], test_row(3, 4.0)).col1[2].col0", BIGINT, 32); }