new Keys.ExpressionKeys<>(new int[]{0}, typeInfo), typeInfo, config); TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, typeInfo);
Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldExpression, this.type); fields = ek.computeLogicalKeyPositions(); if (!Keys.ExpressionKeys.isSortKey(fieldExpression, this.type)) { throw new InvalidProgramException("Selected sort key is not a sortable type");
Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldExpression, this.type); fields = ek.computeLogicalKeyPositions(); if (!Keys.ExpressionKeys.isSortKey(fieldExpression, this.type)) { throw new InvalidProgramException("Selected sort key is not a sortable type");
Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldExpression, this.type); fields = ek.computeLogicalKeyPositions(); if (!Keys.ExpressionKeys.isSortKey(fieldExpression, this.type)) { throw new InvalidProgramException("Selected sort key is not a sortable type");
Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(field, this.type); int[] flatKeys = ek.computeLogicalKeyPositions(); if (!Keys.ExpressionKeys.isSortKey(field, this.type)) { throw new InvalidProgramException("Selected sort key is not a sortable type");
Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(field, this.type); int[] flatKeys = ek.computeLogicalKeyPositions(); if (!Keys.ExpressionKeys.isSortKey(field, this.type)) { throw new InvalidProgramException("Selected sort key is not a sortable type");
/** * Sorts {@link org.apache.flink.api.java.tuple.Tuple} elements within a group on the specified field in the specified {@link Order}. * * <p><b>Note: Only groups of Tuple or Pojo elements can be sorted.</b> * * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(int, Order)} calls. * * @param field The Tuple field on which the group is sorted. * @param order The Order in which the specified Tuple field is sorted. * @return A SortedGrouping with specified order of group element. * * @see org.apache.flink.api.java.tuple.Tuple * @see Order */ public SortedGrouping<T> sortGroup(int field, Order order) { if (groupSortSelectorFunctionKey != null) { throw new InvalidProgramException("Chaining sortGroup with KeySelector sorting is not supported"); } if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); addSortGroupInternal(ek, order); return this; }
private int[] getAllFlatKeys(String[] fieldExpressions) { int[] allKeys = null; for (String keyExp : fieldExpressions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type); int[] flatKeys = ek.computeLogicalKeyPositions(); if (allKeys == null) { allKeys = flatKeys; } else { // check for duplicates for (int key1 : flatKeys) { for (int key2 : allKeys) { if (key1 == key2) { throw new InvalidProgramException("Duplicate fields in field expression " + keyExp); } } } // append flat keys int oldLength = allKeys.length; int newLength = oldLength + flatKeys.length; allKeys = Arrays.copyOf(allKeys, newLength); System.arraycopy(flatKeys, 0, allKeys, oldLength, flatKeys.length); } } return allKeys; }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }
/** * Sorts {@link org.apache.flink.api.java.tuple.Tuple} or POJO elements within a group on the specified field in the specified {@link Order}. * * <p><b>Note: Only groups of Tuple or Pojo elements can be sorted.</b> * * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(String, Order)} calls. * * @param field The Tuple or Pojo field on which the group is sorted. * @param order The Order in which the specified field is sorted. * @return A SortedGrouping with specified order of group element. * * @see org.apache.flink.api.java.tuple.Tuple * @see Order */ public SortedGrouping<T> sortGroup(String field, Order order) { if (groupSortSelectorFunctionKey != null) { throw new InvalidProgramException("Chaining sortGroup with KeySelector sorting is not supported"); } if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); addSortGroupInternal(ek, order); return this; }
private int[] getAllFlatKeys(String[] fieldExpressions) { int[] allKeys = null; for (String keyExp : fieldExpressions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type); int[] flatKeys = ek.computeLogicalKeyPositions(); if (allKeys == null) { allKeys = flatKeys; } else { // check for duplicates for (int key1 : flatKeys) { for (int key2 : allKeys) { if (key1 == key2) { throw new InvalidProgramException("Duplicate fields in field expression " + keyExp); } } } // append flat keys int oldLength = allKeys.length; int newLength = oldLength + flatKeys.length; allKeys = Arrays.copyOf(allKeys, newLength); System.arraycopy(flatKeys, 0, allKeys, oldLength, flatKeys.length); } } return allKeys; }
/** * Sorts {@link org.apache.flink.api.java.tuple.Tuple} or POJO elements within a group on the specified field in the specified {@link Order}. * * <p><b>Note: Only groups of Tuple or Pojo elements can be sorted.</b> * * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(String, Order)} calls. * * @param field The Tuple or Pojo field on which the group is sorted. * @param order The Order in which the specified field is sorted. * @return A SortedGrouping with specified order of group element. * * @see org.apache.flink.api.java.tuple.Tuple * @see Order */ public SortedGrouping<T> sortGroup(String field, Order order) { if (groupSortSelectorFunctionKey != null) { throw new InvalidProgramException("Chaining sortGroup with KeySelector sorting is not supported"); } if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); addSortGroupInternal(ek, order); return this; }
/** * Sorts {@link org.apache.flink.api.java.tuple.Tuple} elements within a group on the specified field in the specified {@link Order}. * * <p><b>Note: Only groups of Tuple or Pojo elements can be sorted.</b> * * <p>Groups can be sorted by multiple fields by chaining {@link #sortGroup(int, Order)} calls. * * @param field The Tuple field on which the group is sorted. * @param order The Order in which the specified Tuple field is sorted. * @return A SortedGrouping with specified order of group element. * * @see org.apache.flink.api.java.tuple.Tuple * @see Order */ public SortedGrouping<T> sortGroup(int field, Order order) { if (groupSortSelectorFunctionKey != null) { throw new InvalidProgramException("Chaining sortGroup with KeySelector sorting is not supported"); } if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); addSortGroupInternal(ek, order); return this; }
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor( Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) { TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); int[] sortKeyPositions = newKey.computeLogicalKeyPositions(); Ordering partitionOrdering = new Ordering(); for (int keyPosition : sortKeyPositions) { partitionOrdering.appendOrdering(keyPosition, null, order); } // distinguish between partition types UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey); SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name); noop.setInput(keyedInput); if (this.getParallelism() < 0) { // use parallelism of input if not explicitly specified noop.setParallelism(input.getParallelism()); } else { // use explicitly specified parallelism noop.setParallelism(this.getParallelism()); } return KeyFunctions.appendKeyRemover(noop, keys); }
private void addSortGroupInternal(ExpressionKeys<T> ek, Order order) { Preconditions.checkArgument(order != null, "Order can not be null"); int[] additionalKeyPositions = ek.computeLogicalKeyPositions(); int newLength = this.groupSortKeyPositions.length + additionalKeyPositions.length; this.groupSortKeyPositions = Arrays.copyOf(this.groupSortKeyPositions, newLength); this.groupSortOrders = Arrays.copyOf(this.groupSortOrders, newLength); int pos = newLength - additionalKeyPositions.length; int off = newLength - additionalKeyPositions.length; for (; pos < newLength; pos++) { this.groupSortKeyPositions[pos] = additionalKeyPositions[pos - off]; this.groupSortOrders[pos] = order; // use the same order } } }
private void addSortGroupInternal(ExpressionKeys<T> ek, Order order) { Preconditions.checkArgument(order != null, "Order can not be null"); int[] additionalKeyPositions = ek.computeLogicalKeyPositions(); int newLength = this.groupSortKeyPositions.length + additionalKeyPositions.length; this.groupSortKeyPositions = Arrays.copyOf(this.groupSortKeyPositions, newLength); this.groupSortOrders = Arrays.copyOf(this.groupSortOrders, newLength); int pos = newLength - additionalKeyPositions.length; int off = newLength - additionalKeyPositions.length; for (; pos < newLength; pos++) { this.groupSortKeyPositions[pos] = additionalKeyPositions[pos - off]; this.groupSortOrders[pos] = order; // use the same order } } }
private void ensureSortableKey(int field) throws InvalidProgramException { if (!Keys.ExpressionKeys.isSortKey(field, getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } }
private int[] getAllFlatKeys(int[] fieldPositions) { Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(fieldPositions, this.type); return ek.computeLogicalKeyPositions(); }
public SortedGrouping(DataSet<T> set, Keys<T> keys, String field, Order order) { super(set, keys); if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } // resolve String-field to int using the expression keys ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); this.groupSortKeyPositions = ek.computeLogicalKeyPositions(); this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); // if field == "*" }
public SortedGrouping(DataSet<T> set, Keys<T> keys, int field, Order order) { super(set, keys); if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) { throw new InvalidProgramException("Selected sort key is not a sortable type"); } // use int-based expression key to properly resolve nested tuples for grouping ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType()); this.groupSortKeyPositions = ek.computeLogicalKeyPositions(); this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); }