public HistogramStateSerializer(Type type) { this.type = type; this.serializedType = new MapType(type, BigintType.BIGINT); }
@Override public List<Type> getTypeParameters() { return ImmutableList.of(getKeyType(), getValueType()); }
@Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries) { return createBlockBuilder(blockBuilderStatus, expectedEntries, EXPECTED_BYTES_PER_ENTRY); }
private static Map<Double, Double> extractSingleValue(Block block) throws IOException { MapType mapType = new MapType(DOUBLE, DOUBLE); return (Map<Double, Double>) mapType.getObjectValue(null, block, 0); }
@Test public void testMapDisplayName() { MapType mapType = new MapType(BIGINT, VarcharType.createVarcharType(42)); assertEquals(mapType.getDisplayName(), "map(bigint, varchar(42))"); mapType = new MapType(BIGINT, VARCHAR); assertEquals(mapType.getDisplayName(), "map(bigint, varchar)"); } }
@Test public void testArrayHistograms() throws Exception { ArrayType arrayType = new ArrayType(VARCHAR); MapType mapType = new MapType(arrayType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), arrayType.getTypeSignature().toString())); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableList.of("a", "b", "c"), 1L, ImmutableList.of("d", "e", "f"), 1L, ImmutableList.of("c", "b", "a"), 1L), createStringArraysBlock(ImmutableList.of(ImmutableList.of("a", "b", "c"), ImmutableList.of("d", "e", "f"), ImmutableList.of("c", "b", "a")))); }
@Test public void testMapHistograms() throws Exception { MapType innerMapType = new MapType(VARCHAR, VARCHAR); MapType mapType = new MapType(innerMapType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), innerMapType.getTypeSignature().toString())); BlockBuilder builder = innerMapType.createBlockBuilder(new BlockBuilderStatus(), 3); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("a", "b"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("c", "d"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("e", "f"))); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableMap.of("a", "b"), 1L, ImmutableMap.of("c", "d"), 1L, ImmutableMap.of("e", "f"), 1L), builder.build()); }
public static Block createMap(MapType mapType, Block keyBlock, Block valueBlock) { BlockBuilder blockBuilder = new InterleavedBlockBuilder(mapType.getTypeParameters(), new BlockBuilderStatus(), keyBlock.getPositionCount() * 2); checkCondition(keyBlock.getPositionCount() == valueBlock.getPositionCount(), INVALID_FUNCTION_ARGUMENT, "Key and value arrays must be the same length"); for (int i = 0; i < keyBlock.getPositionCount(); i++) { if (keyBlock.isNull(i)) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "map key cannot be null"); } mapType.getKeyType().appendTo(keyBlock, i, blockBuilder); mapType.getValueType().appendTo(valueBlock, i, blockBuilder); } return blockBuilder.build(); } }
if (mapType.getValueType() != null && !UnknownType.UNKNOWN.equals(mapType.getValueType())) { schema = new LogicalSchema(); schema.addField(fromPrestoTypeToPigSchema(null, mapType.getValueType()));
@Override public void serialize(KeyValuePairsState state, BlockBuilder out) { if (state.get() == null) { out.appendNull(); } else { mapType.writeObject(out, state.get().serialize()); } }
@Test public void testLargerHistograms() throws Exception { MapType mapType = new MapType(VARCHAR, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.VARCHAR)); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of("a", 25L, "b", 10L, "c", 12L, "d", 1L, "e", 2L), createStringsBlock("a", "b", "c", "d", "e", "e", "c", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "a", "a", "a", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c", "b", "a", "c", "c")); } }
@Test public void testDoubleMapMap() throws Exception { MapType innerMapType = new MapType(VARCHAR, VARCHAR); MapType mapType = new MapType(DOUBLE, innerMapType); InternalAggregationFunction aggFunc = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.DOUBLE, innerMapType.getTypeSignature().toString())); BlockBuilder builder = innerMapType.createBlockBuilder(new BlockBuilderStatus(), 3); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("a", "b"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("c", "d"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("e", "f"))); assertAggregation( aggFunc, 1.0, ImmutableMap.of(1.0, ImmutableMap.of("a", "b"), 2.0, ImmutableMap.of("c", "d"), 3.0, ImmutableMap.of("e", "f")), createDoublesBlock(1.0, 2.0, 3.0), builder.build()); }
private static <K, V> void testMultimapAgg(Type keyType, List<K> expectedKeys, Type valueType, List<V> expectedValues) { checkState(expectedKeys.size() == expectedValues.size(), "expectedKeys and expectedValues should have equal size"); MapType mapType = new MapType(keyType, new ArrayType(valueType)); Signature signature = new Signature(NAME, AGGREGATE, mapType.getTypeSignature(), keyType.getTypeSignature(), valueType.getTypeSignature()); InternalAggregationFunction aggFunc = metadata.getFunctionRegistry().getAggregateFunctionImplementation(signature); Map<K, List<V>> map = new HashMap<>(); for (int i = 0; i < expectedKeys.size(); i++) { if (!map.containsKey(expectedKeys.get(i))) { map.put(expectedKeys.get(i), new ArrayList<>()); } map.get(expectedKeys.get(i)).add(expectedValues.get(i)); } RowPageBuilder builder = RowPageBuilder.rowPageBuilder(keyType, valueType); for (int i = 0; i < expectedKeys.size(); i++) { builder.row(expectedKeys.get(i), expectedValues.get(i)); } assertAggregation(aggFunc, 1.0, map.isEmpty() ? null : map, builder.build().getBlocks()); } }
public KeyValuePairStateSerializer(Type keyType, Type valueType, boolean isMultiValue) { this.mapType = new MapType(keyType, valueType); this.isMultiValue = isMultiValue; }
public MapUnnester(MapType mapType, @Nullable Block mapBlock) { this.channelCount = 2; requireNonNull(mapType, "mapType is null"); this.keyType = mapType.getKeyType(); this.valueType = mapType.getValueType(); this.block = mapBlock; this.positionCount = mapBlock == null ? 0 : mapBlock.getPositionCount(); }
@Test public void testDoubleArrayMap() throws Exception { ArrayType arrayType = new ArrayType(VARCHAR); MapType mapType = new MapType(DOUBLE, arrayType); InternalAggregationFunction aggFunc = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), StandardTypes.DOUBLE, arrayType.getTypeSignature().toString())); assertAggregation( aggFunc, 1.0, ImmutableMap.of(1.0, ImmutableList.of("a", "b"), 2.0, ImmutableList.of("c", "d"), 3.0, ImmutableList.of("e", "f")), createDoublesBlock(1.0, 2.0, 3.0), createStringArraysBlock(ImmutableList.of(ImmutableList.of("a", "b"), ImmutableList.of("c", "d"), ImmutableList.of("e", "f")))); }
private Type toPrestoMapType(final LogicalSchema.LogicalFieldSchema field) { final Type key = StringType.STRING; Type value = UnknownType.UNKNOWN; if (field.schema != null) { final List<LogicalSchema.LogicalFieldSchema> fields = field.schema.getFields(); if (fields.size() > 0) { value = toPrestoType(fields.get(0)); } } return new MapType(key, value); } }
public static boolean canCastFromJson(Type type) { String baseType = type.getTypeSignature().getBase(); if (baseType.equals(StandardTypes.BOOLEAN) || baseType.equals(StandardTypes.BIGINT) || baseType.equals(StandardTypes.DOUBLE) || baseType.equals(StandardTypes.VARCHAR) || baseType.equals(StandardTypes.JSON)) { return true; } if (type instanceof ArrayType) { return canCastFromJson(((ArrayType) type).getElementType()); } if (type instanceof MapType) { return isValidJsonObjectKeyType(((MapType) type).getKeyType()) && canCastFromJson(((MapType) type).getValueType()); } return false; }
@Test public void testArrayDoubleMap() throws Exception { ArrayType arrayType = new ArrayType(VARCHAR); MapType mapType = new MapType(arrayType, DOUBLE); InternalAggregationFunction aggFunc = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature( NAME, AGGREGATE, mapType.getTypeSignature().toString(), arrayType.getTypeSignature().toString(), StandardTypes.DOUBLE )); assertAggregation( aggFunc, 1.0, ImmutableMap.of( ImmutableList.of("a", "b"), 1.0, ImmutableList.of("c", "d"), 2.0, ImmutableList.of("e", "f"), 3.0), createStringArraysBlock(ImmutableList.of(ImmutableList.of("a", "b"), ImmutableList.of("c", "d"), ImmutableList.of("e", "f"))), createDoublesBlock(1.0, 2.0, 3.0)); } }
MapType myMapType = new MapType();