private void addSortGroupInternal(ExpressionKeys<T> ek, Order order) { Preconditions.checkArgument(order != null, "Order can not be null"); int[] additionalKeyPositions = ek.computeLogicalKeyPositions(); int newLength = this.groupSortKeyPositions.length + additionalKeyPositions.length; this.groupSortKeyPositions = Arrays.copyOf(this.groupSortKeyPositions, newLength); this.groupSortOrders = Arrays.copyOf(this.groupSortOrders, newLength); int pos = newLength - additionalKeyPositions.length; int off = newLength - additionalKeyPositions.length; for (; pos < newLength; pos++) { this.groupSortKeyPositions[pos] = additionalKeyPositions[pos - off]; this.groupSortOrders[pos] = order; // use the same order } } }
private int[] getAllFlatKeys(String[] fieldExpressions) { int[] allKeys = null; for (String keyExp : fieldExpressions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type); int[] flatKeys = ek.computeLogicalKeyPositions(); if (allKeys == null) { allKeys = flatKeys; } else { // check for duplicates for (int key1 : flatKeys) { for (int key2 : allKeys) { if (key1 == key2) { throw new InvalidProgramException("Duplicate fields in field expression " + keyExp); } } } // append flat keys int oldLength = allKeys.length; int newLength = oldLength + flatKeys.length; allKeys = Arrays.copyOf(allKeys, newLength); System.arraycopy(flatKeys, 0, allKeys, oldLength, flatKeys.length); } } return allKeys; }
int[] flatKeys = ek.computeLogicalKeyPositions();
private int[] getAllFlatKeys(int[] fieldPositions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldPositions, this.type); return ek.computeLogicalKeyPositions(); }
public SortedGrouping(DataSet<T> set, Keys<T> keys, String field, Order order) { super(set, keys); if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } // resolve String-field to int using the expression keys ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); this.groupSortKeyPositions = ek.computeLogicalKeyPositions(); this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); // if field == "*" }
ExpressionKeys<Tuple3<String, Tuple3<String, String, String>, String>> fpk = new ExpressionKeys<>(0, typeInfo); Assert.assertArrayEquals(new int[] {0}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {1,2,3}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,4}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {4,0}, fpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0}, complexFpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions()); Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6}, complexFpk.computeLogicalKeyPositions());
fields = ek.computeLogicalKeyPositions();
Assert.fail("Keys must be compatible: "+e.getMessage()); TypeComparator<TestUserClass> multiPojoComp = pType.createComparator( expressKey.computeLogicalKeyPositions(), new boolean[]{true, true, true}, 0, new ExecutionConfig()); int multiPojoHash = multiPojoComp.hash(pojoTestRecord); TypeComparator<Tuple3<Integer, String, Double>> multiTupleComp = multiTupleType.createComparator(fieldKey.computeLogicalKeyPositions(), new boolean[] {true, true,true}, 0, new ExecutionConfig()); int multiTupleHash = multiTupleComp.hash(multiTupleTest);
String keyExp = orderKeysA[i]; Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type); int[] flatKeys = ek.computeLogicalKeyPositions();
@Test public void testStandardTupleKeys() { TupleTypeInfo<Tuple7<String, String, String, String, String, String, String>> typeInfo = new TupleTypeInfo<>( BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,BasicTypeInfo.STRING_TYPE_INFO); ExpressionKeys<Tuple7<String, String, String, String, String, String, String>> ek; for( int i = 1; i < 8; i++) { int[] ints = new int[i]; for( int j = 0; j < i; j++) { ints[j] = j; } int[] inInts = Arrays.copyOf(ints, ints.length); // copy, just to make sure that the code is not cheating by changing the ints. ek = new ExpressionKeys<>(inInts, typeInfo); Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions()); Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length); ArrayUtils.reverse(ints); inInts = Arrays.copyOf(ints, ints.length); ek = new ExpressionKeys<>(inInts, typeInfo); Assert.assertArrayEquals(ints, ek.computeLogicalKeyPositions()); Assert.assertEquals(ints.length, ek.computeLogicalKeyPositions().length); } }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }
@Test public void testTupleWithNestedPojo() { TypeInformation<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ti = new TupleTypeInfo<>( BasicTypeInfo.INT_TYPE_INFO, TypeExtractor.getForClass(Pojo1.class), TypeExtractor.getForClass(PojoWithMultiplePojos.class) ); ExpressionKeys<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ek; ek = new ExpressionKeys<>(0, ti); Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>(1, ti); Assert.assertArrayEquals(new int[] {1,2}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>(2, ti); Assert.assertArrayEquals(new int[] {3,4,5,6,7}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>(new int[]{}, ti, true); Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6,7}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("*", ti); Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6,7}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("f2.p1.*", ti); Assert.assertArrayEquals(new int[] {4,5}, ek.computeLogicalKeyPositions()); }
@Test public void testPojoKeys() { TypeInformation<PojoWithMultiplePojos> ti = TypeExtractor.getForClass(PojoWithMultiplePojos.class); ExpressionKeys<PojoWithMultiplePojos> ek; ek = new ExpressionKeys<>("*", ti); Assert.assertArrayEquals(new int[] {0,1,2,3,4}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("p1.*", ti); Assert.assertArrayEquals(new int[] {1,2}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("p2.*", ti); Assert.assertArrayEquals(new int[] {3,4}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("p1", ti); Assert.assertArrayEquals(new int[] {1,2}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("p2", ti); Assert.assertArrayEquals(new int[] {3,4}, ek.computeLogicalKeyPositions()); ek = new ExpressionKeys<>("i0", ti); Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions()); }
@Override protected TypeComparator<PojoContainingTuple> createComparator(boolean ascending) { Assert.assertTrue(type instanceof CompositeType); CompositeType<PojoContainingTuple> cType = (CompositeType<PojoContainingTuple>) type; ExpressionKeys<PojoContainingTuple> keys = new ExpressionKeys<PojoContainingTuple>(new String[] {"theTuple.*"}, cType); boolean[] orders = new boolean[keys.getNumberOfKeyFields()]; Arrays.fill(orders, ascending); return cType.createComparator(keys.computeLogicalKeyPositions(), orders, 0, new ExecutionConfig()); }
private int[] getAllFlatKeys(int[] fieldPositions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldPositions, this.type); return ek.computeLogicalKeyPositions(); }
@Override protected TypeComparator<PojoContainingTuple> createComparator(boolean ascending) { Assert.assertTrue(type instanceof CompositeType); CompositeType<PojoContainingTuple> cType = (CompositeType<PojoContainingTuple>) type; ExpressionKeys<PojoContainingTuple> keys = new ExpressionKeys<PojoContainingTuple>(new String[] {"theTuple.*"}, cType); boolean[] orders = new boolean[keys.getNumberOfKeyFields()]; Arrays.fill(orders, ascending); return cType.createComparator(keys.computeLogicalKeyPositions(), orders, 0, new ExecutionConfig()); }
public SortedGrouping(DataSet<T> set, Keys<T> keys, String field, Order order) { super(set, keys); if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } // resolve String-field to int using the expression keys ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); this.groupSortKeyPositions = ek.computeLogicalKeyPositions(); this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); // if field == "*" }
public SortedGrouping(DataSet<T> set, Keys<T> keys, int field, Order order) { super(set, keys); if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } // use int-based expression key to properly resolve nested tuples for grouping ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); this.groupSortKeyPositions = ek.computeLogicalKeyPositions(); this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); }
@Test public void testBasicType() { TypeInformation<Long> longType = BasicTypeInfo.LONG_TYPE_INFO; ExpressionKeys<Long> ek = new ExpressionKeys<>("*", longType); Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions()); }
@Test public void testKeyGenericType() { TypeInformation<GenericKeyType> genericType = new GenericTypeInfo<>(GenericKeyType.class); ExpressionKeys<GenericKeyType> ek = new ExpressionKeys<>("*", genericType); Assert.assertArrayEquals(new int[] {0}, ek.computeLogicalKeyPositions()); }