@Override public Type createType(TypeManager typeManager, List<TypeParameter> parameters) { checkArgument(parameters.size() == 2, "Expected two parameters, got %s", parameters); TypeParameter firstParameter = parameters.get(0); TypeParameter secondParameter = parameters.get(1); checkArgument( firstParameter.getKind() == ParameterKind.TYPE && secondParameter.getKind() == ParameterKind.TYPE, "Expected key and type to be types, got %s", parameters); Type keyType = firstParameter.getType(); Type valueType = secondParameter.getType(); MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, ImmutableList.of(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType)); return new MapType( keyType, valueType, keyBlockNativeEquals, keyBlockEquals, keyNativeHashCode, keyBlockHashCode); } }
private static BlockBuilder createMapBuilder(int expectedEntries) { MethodHandle varcharNativeEquals = MethodHandleUtil.methodHandle(Slice.class, "equals", Object.class).asType(MethodType.methodType(boolean.class, Slice.class, Slice.class)); MethodHandle varcharBlockNativeEquals = compose(varcharNativeEquals, nativeValueGetter(VARCHAR)); MethodHandle varcharBlockEquals = compose(varcharNativeEquals, nativeValueGetter(VARCHAR), nativeValueGetter(VARCHAR)); return new MapBlockBuilder( VARCHAR, VARCHAR, varcharBlockNativeEquals, varcharBlockEquals, MethodHandleUtil.methodHandle(Slice.class, "hashCode").asType(MethodType.methodType(long.class, Slice.class)), MethodHandleUtil.methodHandle(TestColumnarMap.class, "blockVarcharHashCode", Block.class, int.class), null, expectedEntries); }
@Test public void testCompose2withoutS() throws Throwable { MethodHandle fU2R = methodHandle(TestMethodHandleUtil.class, "fU2R", U.class); MethodHandle fT2U = methodHandle(TestMethodHandleUtil.class, "fT2U", T1.class, T2.class); MethodHandle composed = compose(fU2R, fT2U); assertEquals((R) composed.invokeExact(new T1(2), new T2(3)), new R(6)); }
/** * The signature of the returned MethodHandle is (Block fromMap, int position, ConnectorSession session, BlockBuilder mapBlockBuilder)void. * The processor will get the value from fromMap, cast it and write to toBlock. */ private MethodHandle buildProcessor(FunctionRegistry functionRegistry, Type fromType, Type toType, boolean isKey) { MethodHandle getter = nativeValueGetter(fromType); // Adapt cast that takes ([ConnectorSession,] ?) to one that takes (?, ConnectorSession), where ? is the return type of getter. ScalarFunctionImplementation castImplementation = functionRegistry.getScalarFunctionImplementation(functionRegistry.getCoercion(fromType, toType)); MethodHandle cast = castImplementation.getMethodHandle(); if (cast.type().parameterArray()[0] != ConnectorSession.class) { cast = MethodHandles.dropArguments(cast, 0, ConnectorSession.class); } cast = permuteArguments(cast, methodType(cast.type().returnType(), cast.type().parameterArray()[1], cast.type().parameterArray()[0]), 1, 0); MethodHandle target = compose(cast, getter); // If the key cast function is nullable, check the result is not null. if (isKey && castImplementation.isNullable()) { target = compose(nullChecker(target.type().returnType()), target); } MethodHandle writer = nativeValueWriter(toType); writer = permuteArguments(writer, methodType(void.class, writer.type().parameterArray()[1], writer.type().parameterArray()[0]), 1, 0); return compose(writer, target.asType(methodType(unwrap(target.type().returnType()), target.type().parameterArray()))); }
@Test public void testMapDisplayName() { MapType mapType = new MapType( BIGINT, createVarcharType(42), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation")); assertEquals(mapType.getDisplayName(), "map(bigint, varchar(42))"); mapType = new MapType( BIGINT, VARCHAR, MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation"), MethodHandleUtil.methodHandle(TestMapType.class, "throwUnsupportedOperation")); assertEquals(mapType.getDisplayName(), "map(bigint, varchar)"); }
@Test public void testCompose3() throws Throwable { MethodHandle fUV2R = methodHandle(TestMethodHandleUtil.class, "fUV2R", U.class, V.class); MethodHandle fS2U = methodHandle(TestMethodHandleUtil.class, "fS2U", S1.class, S2.class); MethodHandle fT2V = methodHandle(TestMethodHandleUtil.class, "fT2V", T1.class, T2.class); MethodHandle composed = compose(fUV2R, fS2U, fT2V); assertEquals((R) composed.invokeExact(new S1(2), new S2(3), new T1(5), new T2(7)), new R(210)); }
@Test public void testRowDisplayNoColumnNames() { List<Type> types = asList( BOOLEAN, DOUBLE, new ArrayType(VARCHAR), new MapType( BOOLEAN, DOUBLE, methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"))); RowType row = RowType.anonymous(types); assertEquals( row.getDisplayName(), "row(boolean, double, array(varchar), map(boolean, double))"); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Type keyType = TypeSerde.readType(typeManager, sliceInput); MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( format("Deserialized SingleMapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); } return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } }
@Test public void testCompose2() throws Throwable { MethodHandle fUS2R = methodHandle(TestMethodHandleUtil.class, "fUS2R", U.class, S1.class, S2.class); MethodHandle fT2U = methodHandle(TestMethodHandleUtil.class, "fT2U", T1.class, T2.class); MethodHandle composed = compose(fUS2R, fT2U); assertEquals((R) composed.invokeExact(new T1(2), new T2(3), new S1(5), new S2(7)), new R(210)); }
@Test public void testRowDisplayName() { List<RowType.Field> fields = asList( RowType.field("bool_col", BOOLEAN), RowType.field("double_col", DOUBLE), RowType.field("array_col", new ArrayType(VARCHAR)), RowType.field("map_col", new MapType( BOOLEAN, DOUBLE, methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation")))); RowType row = RowType.from(fields); assertEquals( row.getDisplayName(), "row(bool_col boolean, double_col double, array_col array(varchar), map_col map(boolean, double))"); }
private static Block createMapBlock(int positionCount) { Type keyType = VARCHAR; Type valueType = VARCHAR; MethodHandle keyNativeEquals = TYPE_MANAGER.resolveOperator(OperatorType.EQUAL, ImmutableList.of(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = TYPE_MANAGER.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType)); MapType mapType = new MapType( keyType, valueType, keyBlockNativeEquals, keyBlockEquals, keyNativeHashCode, keyBlockHashCode); Block keyBlock = createDictionaryBlock(generateList("key", positionCount)); Block valueBlock = createDictionaryBlock(generateList("value", positionCount)); int[] offsets = new int[positionCount + 1]; int mapSize = keyBlock.getPositionCount() / positionCount; for (int i = 0; i < offsets.length; i++) { offsets[i] = mapSize * i; } return mapType.createBlockFromKeyValue(Optional.empty(), offsets, keyBlock, valueBlock); }
@Test public void testRowDisplayMixedUnnamedColumns() { List<RowType.Field> fields = asList( RowType.field(BOOLEAN), RowType.field("double_col", DOUBLE), RowType.field(new ArrayType(VARCHAR)), RowType.field("map_col", new MapType( BOOLEAN, DOUBLE, methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation"), methodHandle(TestRowType.class, "throwUnsupportedOperation")))); RowType row = RowType.from(fields); assertEquals( row.getDisplayName(), "row(boolean, double_col double, array(varchar), map_col map(boolean, double))"); }
MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = TYPE_MANAGER.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType));
@Test public void testParquetTupleDomainMap() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_map", HiveType.valueOf("map<int,int>"), parseTypeSignature(StandardTypes.MAP), 0, REGULAR, Optional.empty()); MapType mapType = new MapType( INTEGER, INTEGER, methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException")); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(mapType))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_map", new GroupType(REPEATED, "map", new PrimitiveType(REQUIRED, INT32, "key"), new PrimitiveType(OPTIONAL, INT32, "value")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { Type keyType = TypeSerde.readType(typeManager, sliceInput); MethodHandle keyNativeEquals = typeManager.resolveOperator(OperatorType.EQUAL, asList(keyType, keyType)); MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = typeManager.resolveOperator(OperatorType.HASH_CODE, singletonList(keyType)); Block keyBlock = blockEncodingSerde.readBlock(sliceInput); Block valueBlock = blockEncodingSerde.readBlock(sliceInput); int[] hashTable = new int[sliceInput.readInt()]; sliceInput.readBytes(wrappedIntArray(hashTable)); if (keyBlock.getPositionCount() != valueBlock.getPositionCount() || keyBlock.getPositionCount() * HASH_MULTIPLIER != hashTable.length) { throw new IllegalArgumentException( format("Deserialized SingleMapBlock violates invariants: key %d, value %d, hash %d", keyBlock.getPositionCount(), valueBlock.getPositionCount(), hashTable.length)); } return new SingleMapBlock(0, keyBlock.getPositionCount() * 2, keyBlock, valueBlock, hashTable, keyType, keyNativeHashCode, keyBlockNativeEquals); } }
@Test public void testParquetTupleDomainMap() { HiveColumnHandle columnHandle = new HiveColumnHandle("my_map", HiveType.valueOf("map<int,int>"), parseTypeSignature(StandardTypes.MAP), 0, REGULAR, Optional.empty()); MapType mapType = new MapType( INTEGER, INTEGER, methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"), methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException")); TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(mapType))); MessageType fileSchema = new MessageType("hive_schema", new GroupType(OPTIONAL, "my_map", new GroupType(REPEATED, "map", new PrimitiveType(REQUIRED, INT32, "key"), new PrimitiveType(OPTIONAL, INT32, "value")))); Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain); assertTrue(tupleDomain.getDomains().get().isEmpty()); }
MethodHandle keyBlockNativeEquals = compose(keyNativeEquals, nativeValueGetter(keyType)); MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType)); MethodHandle keyNativeHashCode = TYPE_MANAGER.resolveOperator(OperatorType.HASH_CODE, ImmutableList.of(keyType)); MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType));