private static void appendTypedArray(Block array, Type type, TypedSet typedSet, BlockBuilder blockBuilder) { for (int i = 0; i < array.getPositionCount(); i++) { if (!typedSet.contains(array, i)) { typedSet.add(array, i); type.appendTo(array, i, blockBuilder); } } }
private static void testBigintFor(TypedSet typedSet, Block longBlock) { Set<Long> set = new HashSet<>(); for (int blockPosition = 0; blockPosition < longBlock.getPositionCount(); blockPosition++) { long number = BIGINT.getLong(longBlock, blockPosition); assertEquals(typedSet.contains(longBlock, blockPosition), set.contains(number)); assertEquals(typedSet.size(), set.size()); set.add(number); typedSet.add(longBlock, blockPosition); assertEquals(typedSet.contains(longBlock, blockPosition), set.contains(number)); assertEquals(typedSet.size(), set.size()); } } }
private void testGetElementPositionRandomFor(TypedSet set) { BlockBuilder keys = VARCHAR.createBlockBuilder(null, 5); VARCHAR.writeSlice(keys, utf8Slice("hello")); VARCHAR.writeSlice(keys, utf8Slice("bye")); VARCHAR.writeSlice(keys, utf8Slice("abc")); for (int i = 0; i < keys.getPositionCount(); i++) { set.add(keys, i); } BlockBuilder values = VARCHAR.createBlockBuilder(null, 5); VARCHAR.writeSlice(values, utf8Slice("bye")); VARCHAR.writeSlice(values, utf8Slice("abc")); VARCHAR.writeSlice(values, utf8Slice("hello")); VARCHAR.writeSlice(values, utf8Slice("bad")); values.appendNull(); assertEquals(set.positionOf(values, 4), -1); assertEquals(set.positionOf(values, 2), 0); assertEquals(set.positionOf(values, 1), 2); assertEquals(set.positionOf(values, 0), 1); assertFalse(set.contains(values, 3)); set.add(values, 4); assertTrue(set.contains(values, 4)); }
BlockBuilder blockBuilder = mapBlockBuilder.beginBlockEntry(); for (int i = 0; i < fromMap.getPositionCount(); i += 2) { if (!typedSet.contains(keyBlock, i / 2)) { typedSet.add(keyBlock, i / 2); toKeyType.appendTo(keyBlock, i / 2, blockBuilder);
@TypeParameter("E") @SqlType("array(E)") public static Block except( @TypeParameter("E") Type type, @SqlType("array(E)") Block leftArray, @SqlType("array(E)") Block rightArray) { int leftPositionCount = leftArray.getPositionCount(); int rightPositionCount = rightArray.getPositionCount(); if (leftPositionCount == 0) { return leftArray; } TypedSet typedSet = new TypedSet(type, leftPositionCount + rightPositionCount, "array_except"); BlockBuilder distinctElementBlockBuilder = type.createBlockBuilder(null, leftPositionCount); for (int i = 0; i < rightPositionCount; i++) { typedSet.add(rightArray, i); } for (int i = 0; i < leftPositionCount; i++) { if (!typedSet.contains(leftArray, i)) { typedSet.add(leftArray, i); type.appendTo(leftArray, i, distinctElementBlockBuilder); } } return distinctElementBlockBuilder.build(); } }
if (!typedSet.contains(array, i)) { typedSet.add(array, i); distinctCount++;
if (uniqueKeys.contains(rowBlock, 0)) { mapBlockBuilder.closeEntry(); pageBuilder.declarePosition();
@ScalarFunction @SqlType("array(varchar)") public static Block oldArrayDistinct(@SqlType("array(varchar)") Block array) { if (array.getPositionCount() == 0) { return array; } TypedSet typedSet = new TypedSet(VARCHAR, array.getPositionCount(), "old_array_distinct"); BlockBuilder distinctElementBlockBuilder = VARCHAR.createBlockBuilder(null, array.getPositionCount()); for (int i = 0; i < array.getPositionCount(); i++) { if (!typedSet.contains(array, i)) { typedSet.add(array, i); VARCHAR.appendTo(array, i, distinctElementBlockBuilder); } } return distinctElementBlockBuilder.build(); } }
if (rightTypedSet.contains(leftArray, i)) { intersectTypedSet.add(leftArray, i);
if (!keySet.contains(key, keyValueIndex)) { keySet.add(key, keyValueIndex); keyType.appendTo(key, keyValueIndex, distinctKeyBlockBuilder);
if (keySet.contains(rowBlock, 0)) { entryIndicesList[keySet.positionOf(rowBlock, 0)].add(i);
private static void testBigint(Block longBlock, int expectedSetSize) { TypedSet typedSet = new TypedSet(BIGINT, expectedSetSize); Set<Long> set = new HashSet<>(); for (int blockPosition = 0; blockPosition < longBlock.getPositionCount(); blockPosition++) { long number = BIGINT.getLong(longBlock, blockPosition); assertEquals(typedSet.contains(longBlock, blockPosition), set.contains(number)); assertEquals(typedSet.size(), set.size()); set.add(number); typedSet.add(longBlock, blockPosition); assertEquals(typedSet.contains(longBlock, blockPosition), set.contains(number)); assertEquals(typedSet.size(), set.size()); } } }
/** * Only add this key value pair if we are in multi-value mode or we haven't met this key before. * Otherwise, ignore it. */ public void add(Block key, Block value, int keyPosition, int valuePosition) { if (isMultiValue || !keySet.contains(key, keyPosition)) { keySet.add(key, keyPosition); keyType.appendTo(key, keyPosition, keyBlockBuilder); if (value.isNull(valuePosition)) { valueBlockBuilder.appendNull(); } else { valueType.appendTo(value, valuePosition, valueBlockBuilder); } } } }
public static Block mapConcat(Type keyType, Type valueType, Block leftMap, Block rightMap) { TypedSet typedSet = new TypedSet(keyType, rightMap.getPositionCount()); BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(keyType, valueType), new BlockBuilderStatus(), leftMap.getPositionCount() + rightMap.getPositionCount()); for (int i = 0; i < rightMap.getPositionCount(); i += 2) { typedSet.add(rightMap, i); keyType.appendTo(rightMap, i, blockBuilder); valueType.appendTo(rightMap, i + 1, blockBuilder); } for (int i = 0; i < leftMap.getPositionCount(); i += 2) { if (!typedSet.contains(leftMap, i)) { keyType.appendTo(leftMap, i, blockBuilder); valueType.appendTo(leftMap, i + 1, blockBuilder); } } return blockBuilder.build(); } }
@TypeParameter("E") @SqlType("array(E)") public static Block distinct( @OperatorDependency(operator = OperatorType.EQUAL, returnType = StandardTypes.BOOLEAN, argumentTypes = {"E", "E"}) MethodHandle equals, @TypeParameter("E") Type type, @SqlType("array(E)") Block array) { if (array.getPositionCount() < 2) { return array; } if (array.getPositionCount() == 2) { if (type.equalTo(array, 0, array, 1)) { return array.getSingleValueBlock(0); } else { return array; } } TypedSet typedSet = new TypedSet(type, array.getPositionCount()); BlockBuilder distinctElementBlockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), array.getPositionCount()); for (int i = 0; i < array.getPositionCount(); i++) { if (!typedSet.contains(array, i)) { typedSet.add(array, i); type.appendTo(array, i, distinctElementBlockBuilder); } } return distinctElementBlockBuilder.build(); }
@ScalarFunction @SqlType("array(varchar)") public static Block oldArrayDistinct(@SqlType("array(varchar)") Block array) { if (array.getPositionCount() == 0) { return array; } TypedSet typedSet = new TypedSet(VARCHAR, array.getPositionCount()); BlockBuilder distinctElementBlockBuilder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), array.getPositionCount()); for (int i = 0; i < array.getPositionCount(); i++) { if (!typedSet.contains(array, i)) { typedSet.add(array, i); VARCHAR.appendTo(array, i, distinctElementBlockBuilder); } } return distinctElementBlockBuilder.build(); } }
TypedSet keySet = new TypedSet(keyType, keys.getPositionCount()); for (int keyValueIndex = 0; keyValueIndex < keys.getPositionCount(); keyValueIndex++) { if (!keySet.contains(keys, keyValueIndex)) { keySet.add(keys, keyValueIndex); keyType.appendTo(keys, keyValueIndex, distinctKeyBlockBuilder);