public TaggedValue getInput2AsTaggedValue() { final int[] groupedKeys; if (keys2 != null) { groupedKeys = keys2.computeLogicalKeyPositions(); } else { groupedKeys = null; } return convertTypeInfoToTaggedValue(Input.INPUT_2, in2Type, "", null, groupedKeys); }
public static <X> KeySelector<X, Tuple> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo, ExecutionConfig executionConfig) { if (!(typeInfo instanceof CompositeType)) { throw new InvalidTypesException( "This key operation requires a composite type such as Tuples, POJOs, or Case Classes."); } CompositeType<X> compositeType = (CompositeType<X>) typeInfo; int[] logicalKeyPositions = keys.computeLogicalKeyPositions(); int numKeyFields = logicalKeyPositions.length; TypeInformation<?>[] typeInfos = keys.getKeyFieldTypes(); // use ascending order here, the code paths for that are usually a slight bit faster boolean[] orders = new boolean[numKeyFields]; for (int i = 0; i < numKeyFields; i++) { orders[i] = true; } TypeComparator<X> comparator = compositeType.createComparator(logicalKeyPositions, orders, 0, executionConfig); return new ComparableKeySelector<>(comparator, numKeyFields, new TupleTypeInfo<>(typeInfos)); }
public static <X, K> KeySelector<X, K> getSelectorForOneKey( Keys<X> keys, Partitioner<K> partitioner, TypeInformation<X> typeInfo, ExecutionConfig executionConfig) { if (!(typeInfo instanceof CompositeType)) { throw new InvalidTypesException( "This key operation requires a composite type such as Tuples, POJOs, case classes, etc"); } if (partitioner != null) { keys.validateCustomPartitioner(partitioner, null); } CompositeType<X> compositeType = (CompositeType<X>) typeInfo; int[] logicalKeyPositions = keys.computeLogicalKeyPositions(); if (logicalKeyPositions.length != 1) { throw new IllegalArgumentException("There must be exactly 1 key specified"); } TypeComparator<X> comparator = compositeType.createComparator( logicalKeyPositions, new boolean[] { true }, 0, executionConfig); return new OneKeySelector<>(comparator); }
private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) { Ordering ordering = new Ordering(); final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions(); if (orders == null) { for (int key : logicalKeyPositions) { ordering.appendOrdering(key, null, Order.ASCENDING); } } else { final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes(); int index = 0; for (int i = 0; i < originalKeyFieldTypes.length; i++) { final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields(); for (int j = index; j < index + typeTotalFields; j++) { ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]); } index += typeTotalFields; } } return ordering; }
private <P> PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, Partitioner<P> customPartitioner, TypeInformation<P> partitionerTypeInfo, DataDistribution distribution, String partitionLocationName) { super(input, input.getType()); Preconditions.checkNotNull(pMethod); Preconditions.checkArgument(pKeys != null || pMethod == PartitionMethod.REBALANCE, "Partitioning requires keys"); Preconditions.checkArgument(pMethod != PartitionMethod.CUSTOM || customPartitioner != null, "Custom partioning requires a partitioner."); Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition."); if (distribution != null) { Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified."); Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())), "The types of the flat key fields must be equal to the types of the fields of the distribution."); } if (customPartitioner != null) { pKeys.validateCustomPartitioner(customPartitioner, partitionerTypeInfo); } this.pMethod = pMethod; this.pKeys = pKeys; this.partitionLocationName = partitionLocationName; this.customPartitioner = customPartitioner; this.distribution = distribution; }
if (!keys1.areCompatible(keys2)) { throw new InvalidProgramException("The types of the key fields do not match."); int[] positions = keys1.computeLogicalKeyPositions(); ((SolutionSetPlaceHolder<?>) input1).checkJoinKeyFields(positions); } else { int[] positions = keys2.computeLogicalKeyPositions(); ((SolutionSetPlaceHolder<?>) input2).checkJoinKeyFields(positions); } else {
/** * Sets a custom partitioner for the CoGroup operation. The partitioner will be called on the join keys to determine * the partition a key should be assigned to. The partitioner is evaluated on both inputs in the * same way. * * <p>NOTE: A custom partitioner can only be used with single-field CoGroup keys, not with composite CoGroup keys. * * @param partitioner The custom partitioner to be used. * @return This CoGroup operator, to allow for function chaining. */ public CoGroupOperator<I1, I2, OUT> withPartitioner(Partitioner<?> partitioner) { if (partitioner != null) { keys1.validateCustomPartitioner(partitioner, null); keys2.validateCustomPartitioner(partitioner, null); } this.customPartitioner = getInput1().clean(partitioner); return this; }
/** * Check if two sets of keys are compatible to each other (matching types, key counts) */ public boolean areCompatible(Keys<?> other) throws IncompatibleKeysException { TypeInformation<?>[] thisKeyFieldTypes = this.getKeyFieldTypes(); TypeInformation<?>[] otherKeyFieldTypes = other.getKeyFieldTypes(); if (thisKeyFieldTypes.length != otherKeyFieldTypes.length) { throw new IncompatibleKeysException(IncompatibleKeysException.SIZE_MISMATCH_MESSAGE); } else { for (int i = 0; i < thisKeyFieldTypes.length; i++) { if (!thisKeyFieldTypes[i].equals(otherKeyFieldTypes[i])) { throw new IncompatibleKeysException(thisKeyFieldTypes[i], otherKeyFieldTypes[i] ); } } } return true; }
public boolean isEmpty() { return getNumberOfKeyFields() == 0; }
/** * Sets the order of keys for range partitioning. * NOTE: Only valid for {@link PartitionMethod#RANGE}. * * @param orders array of orders for each specified partition key * @return The partitioneOperator with properly set orders for given keys */ @PublicEvolving public PartitionOperator<T> withOrders(Order... orders) { Preconditions.checkState(pMethod == PartitionMethod.RANGE, "Orders cannot be applied for %s partition " + "method", pMethod); Preconditions.checkArgument(pKeys.getOriginalKeyFieldTypes().length == orders.length, "The number of key " + "fields and orders should be the same."); this.orders = orders; return this; }
@Test public void testAreCompatible1() throws Keys.IncompatibleKeysException { TypeInformation<Pojo2> t1 = TypeExtractor.getForClass(Pojo2.class); TypeInformation<Tuple2<Integer, String>> t2 = new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO); Keys<Pojo2> k1 = new Keys.SelectorFunctionKeys<>( new KeySelector1(), t1, BasicTypeInfo.STRING_TYPE_INFO ); Keys<Tuple2<Integer, String>> k2 = new Keys.SelectorFunctionKeys<>( new KeySelector2(), t2, BasicTypeInfo.STRING_TYPE_INFO ); Assert.assertTrue(k1.areCompatible(k2)); Assert.assertTrue(k2.areCompatible(k1)); }
public Grouping(DataSet<T> set, Keys<T> keys) { if (set == null || keys == null) { throw new NullPointerException(); } if (keys.isEmpty()) { throw new InvalidProgramException("The grouping keys must not be empty."); } this.inputDataSet = set; this.keys = keys; }
String name = getName() != null ? getName() : "CoGroup at " + defaultName; try { keys1.areCompatible(keys2); } catch (IncompatibleKeysException e) { throw new InvalidProgramException("The types of the key fields do not match.", e); keys1.areCompatible(keys2); } catch (IncompatibleKeysException e) { throw new InvalidProgramException("The types of the key fields do not match.", e); int[] logicalKeyPositions1 = keys1.computeLogicalKeyPositions(); int[] logicalKeyPositions2 = keys2.computeLogicalKeyPositions();
private <P> PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, Partitioner<P> customPartitioner, TypeInformation<P> partitionerTypeInfo, DataDistribution distribution, String partitionLocationName) { super(input, input.getType()); Preconditions.checkNotNull(pMethod); Preconditions.checkArgument(pKeys != null || pMethod == PartitionMethod.REBALANCE, "Partitioning requires keys"); Preconditions.checkArgument(pMethod != PartitionMethod.CUSTOM || customPartitioner != null, "Custom partioning requires a partitioner."); Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition."); if (distribution != null) { Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified."); Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())), "The types of the flat key fields must be equal to the types of the fields of the distribution."); } if (customPartitioner != null) { pKeys.validateCustomPartitioner(customPartitioner, partitionerTypeInfo); } this.pMethod = pMethod; this.pKeys = pKeys; this.partitionLocationName = partitionLocationName; this.customPartitioner = customPartitioner; this.distribution = distribution; }
private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) { Ordering ordering = new Ordering(); final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions(); if (orders == null) { for (int key : logicalKeyPositions) { ordering.appendOrdering(key, null, Order.ASCENDING); } } else { final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes(); int index = 0; for (int i = 0; i < originalKeyFieldTypes.length; i++) { final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields(); for (int j = index; j < index + typeTotalFields; j++) { ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]); } index += typeTotalFields; } } return ordering; }
/** * Sets a custom partitioner for this join. The partitioner will be called on the join keys to determine * the partition a key should be assigned to. The partitioner is evaluated on both join inputs in the * same way. * * <p>NOTE: A custom partitioner can only be used with single-field join keys, not with composite join keys. * * @param partitioner The custom partitioner to be used. * @return This join operator, to allow for function chaining. */ public JoinOperator<I1, I2, OUT> withPartitioner(Partitioner<?> partitioner) { if (partitioner != null) { keys1.validateCustomPartitioner(partitioner, null); keys2.validateCustomPartitioner(partitioner, null); } this.customPartitioner = getInput1().clean(partitioner); return this; }
/** * Check if two sets of keys are compatible to each other (matching types, key counts) */ public boolean areCompatible(Keys<?> other) throws IncompatibleKeysException { TypeInformation<?>[] thisKeyFieldTypes = this.getKeyFieldTypes(); TypeInformation<?>[] otherKeyFieldTypes = other.getKeyFieldTypes(); if (thisKeyFieldTypes.length != otherKeyFieldTypes.length) { throw new IncompatibleKeysException(IncompatibleKeysException.SIZE_MISMATCH_MESSAGE); } else { for (int i = 0; i < thisKeyFieldTypes.length; i++) { if (!thisKeyFieldTypes[i].equals(otherKeyFieldTypes[i])) { throw new IncompatibleKeysException(thisKeyFieldTypes[i], otherKeyFieldTypes[i] ); } } } return true; }
public <K> SortedGrouping(DataSet<T> set, Keys<T> keys, Keys.SelectorFunctionKeys<T, K> keySelector, Order order) { super(set, keys); if (!(this.keys instanceof Keys.SelectorFunctionKeys)) { throw new InvalidProgramException("Sorting on KeySelector keys only works with KeySelector grouping."); } TypeInformation<?> sortKeyType = keySelector.getKeyType(); if (!sortKeyType.isSortKeyType()) { throw new InvalidProgramException("Key type " + sortKeyType + " is not sortable."); } this.groupSortKeyPositions = keySelector.computeLogicalKeyPositions(); for (int i = 0; i < groupSortKeyPositions.length; i++) { groupSortKeyPositions[i] += this.keys.getNumberOfKeyFields(); } this.groupSortSelectorFunctionKey = keySelector; this.groupSortOrders = new Order[groupSortKeyPositions.length]; Arrays.fill(this.groupSortOrders, order); }
@Test public void testOriginalTypes1() throws Keys.IncompatibleKeysException { TypeInformation<Tuple2<Integer, String>> t2 = new TupleTypeInfo<>( BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO ); Keys<Tuple2<Integer, String>> k = new Keys.SelectorFunctionKeys<>( new KeySelector2(), t2, BasicTypeInfo.STRING_TYPE_INFO ); Assert.assertArrayEquals( new TypeInformation<?>[] { BasicTypeInfo.STRING_TYPE_INFO }, k.getOriginalKeyFieldTypes() ); }
@Test public void testAreCompatible3() throws Keys.IncompatibleKeysException { TypeInformation<String> t1 = BasicTypeInfo.STRING_TYPE_INFO; TypeInformation<Pojo2> t2 = TypeExtractor.getForClass(Pojo2.class); Keys.ExpressionKeys<String> ek1 = new Keys.ExpressionKeys<>("*", t1); Keys<Pojo2> sk2 = new Keys.SelectorFunctionKeys<>( new KeySelector1(), t2, BasicTypeInfo.STRING_TYPE_INFO ); Assert.assertTrue(sk2.areCompatible(ek1)); }