public GroupCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields if (additionalOrderKeys != null) { for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } }
@Test public void testNewOrdering() { Ordering ordering = new Ordering(); // add a field ordering.appendOrdering(3, Integer.class, Order.ASCENDING); assertEquals(1, ordering.getNumberOfFields()); // add a second field ordering.appendOrdering(1, Long.class, Order.DESCENDING); assertEquals(2, ordering.getNumberOfFields()); // duplicate field index does not change Ordering ordering.appendOrdering(1, String.class, Order.ASCENDING); assertEquals(2, ordering.getNumberOfFields()); // verify field positions, types, and orderings assertArrayEquals(new int[]{3, 1}, ordering.getFieldPositions()); assertArrayEquals(new Class[]{Integer.class, Long.class}, ordering.getTypes()); assertArrayEquals(new boolean[]{true, false}, ordering.getFieldSortDirections()); } }
Ordering newOrdering = new Ordering(); for (int i = 0; i < this.ordering.getInvolvedIndexes().size(); i++) { int targetField = this.ordering.getInvolvedIndexes().get(i); int sourceField = props.getForwardingSourceField(input, targetField); if (sourceField >= 0) { newOrdering.appendOrdering(sourceField, this.ordering.getType(i), this.ordering.getOrder(i)); } else { return null;
protected Ordering getGroupOrdering() { Ordering o = new Ordering(); for (int i = 0; i < this.groupSortKeyPositions.length; i++) { o.appendOrdering(this.groupSortKeyPositions[i], null, this.groupSortOrders[i]); } return o; }
public boolean matchesOrderedPartitioning(Ordering o) { if (this.partitioning == PartitioningProperty.RANGE_PARTITIONED) { if (this.ordering.getNumberOfFields() > o.getNumberOfFields()) { return false; for (int i = 0; i < this.ordering.getNumberOfFields(); i++) { if (!this.ordering.getFieldNumber(i).equals(o.getFieldNumber(i))) { return false; final Order oo = o.getOrder(i); final Order to = this.ordering.getOrder(i); if (oo != Order.NONE) { if (oo == Order.ANY) {
this.splitOrdering = new Ordering(); for (int okey : splitOrdering.getFieldPositions()) { if (key == okey) { throw new InvalidProgramException("Duplicate field in the field expression " + pos); this.splitOrdering.appendOrdering(key, null, orders[i]);
@Override protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) { TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType(); int[] sortColumns = this.partitionOrdering.getFieldPositions(); boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections(); final TypeComparator<IN> sortComparator; if (inputType instanceof CompositeType) { sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig); } else if (inputType instanceof AtomicType) { sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig); } else { throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet."); } Collections.sort(inputData, new Comparator<IN>() { @Override public int compare(IN o1, IN o2) { return sortComparator.compare(o1, o2); } }); return inputData; } }
final TypeComparator<IN2> inputSortComparator2; if (groupOrder1 == null || groupOrder1.getNumberOfFields() == 0) { int[] groupSortKeys = groupOrder1.getFieldPositions(); int[] allSortKeys = new int[inputKeys1.length + groupOrder1.getNumberOfFields()]; System.arraycopy(inputKeys1, 0, allSortKeys, 0, inputKeys1.length); System.arraycopy(groupSortKeys, 0, allSortKeys, inputKeys1.length, groupSortKeys.length); boolean[] groupSortDirections = groupOrder1.getFieldSortDirections(); boolean[] allSortDirections = new boolean[inputKeys1.length + groupSortKeys.length]; Arrays.fill(allSortDirections, 0, inputKeys1.length, true); if (groupOrder2 == null || groupOrder2.getNumberOfFields() == 0) { int[] groupSortKeys = groupOrder2.getFieldPositions(); int[] allSortKeys = new int[inputKeys2.length + groupOrder2.getNumberOfFields()]; System.arraycopy(inputKeys2, 0, allSortKeys, 0, inputKeys2.length); System.arraycopy(groupSortKeys, 0, allSortKeys, inputKeys2.length, groupSortKeys.length); boolean[] groupSortDirections = groupOrder2.getFieldSortDirections(); boolean[] allSortDirections = new boolean[inputKeys2.length + groupSortKeys.length]; Arrays.fill(allSortDirections, 0, inputKeys2.length, true);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { node.setParallelism(in.getSource().getParallelism()); // sorting key info SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode( node, "GroupCombine (" + node.getOperator().getName() + ")", in, // reuse the combine strategy also used in the group reduce DriverStrategy.SORTED_GROUP_COMBINE, this.keyList); // set sorting comparator key info singleInputPlanNode.setDriverKeyInfo(this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections(), 0); // set grouping comparator key info singleInputPlanNode.setDriverKeyInfo(this.keyList, 1); return singleInputPlanNode; }
public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Ordering ordering, Partitioner<?> customPartitioner, DataDistribution distribution) { super(pKeys); Preconditions.checkArgument(pMethod != PartitionMethod.RANGE || pKeys.equals(new FieldSet(ordering.getFieldPositions())), "Partition keys must match the given ordering."); this.pMethod = pMethod; this.customPartitioner = customPartitioner; this.distribution = distribution; this.ordering = ordering; }
public static boolean[] getDirections(Ordering o, int numFields) { final boolean[] dirs = o.getFieldSortDirections(); if (dirs.length == numFields) { return dirs; } else if (dirs.length > numFields) { final boolean[] subSet = new boolean[numFields]; System.arraycopy(dirs, 0, subSet, 0, numFields); return subSet; } else { throw new CompilerException(); } }
public boolean isOrderEqualOnFirstNFields(Ordering other, int n) { if (n > getNumberOfFields() || n > other.getNumberOfFields()) { throw new IndexOutOfBoundsException(); } for (int i = 0; i < n; i++) { final Order o = this.orders.get(i); if (o == Order.NONE || o == Order.ANY || o != other.orders.get(i)) { return false; } } return true; }
@Override public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2, LocalProperties produced1, LocalProperties produced2) { int numRelevantFields = this.keys1.size(); Ordering prod1 = produced1.getOrdering(); Ordering prod2 = produced2.getOrdering(); if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields || prod2.getNumberOfFields() < numRelevantFields) { throw new CompilerException("The given properties do not meet this operators requirements."); } for (int i = 0; i < numRelevantFields; i++) { if (prod1.getOrder(i) != prod2.getOrder(i)) { return false; } } return true; }
@Override public boolean equals(Object obj) { if (obj instanceof RequestedLocalProperties) { final RequestedLocalProperties other = (RequestedLocalProperties) obj; return (this.ordering == other.ordering || (this.ordering != null && this.ordering.equals(other.ordering))) && (this.groupedFields == other.groupedFields || (this.groupedFields != null && this.groupedFields.equals(other.groupedFields))); } else { return false; } }
protected boolean checkSameOrdering(GlobalProperties produced1, GlobalProperties produced2, int numRelevantFields) { Ordering prod1 = produced1.getPartitioningOrdering(); Ordering prod2 = produced2.getPartitioningOrdering(); if (prod1 == null || prod2 == null) { throw new CompilerException("The given properties do not meet this operators requirements."); } // check that order of fields is equivalent if (!checkEquivalentFieldPositionsInKeyFields( prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) { return false; } // check that both inputs have the same directions of order for (int i = 0; i < numRelevantFields; i++) { if (prod1.getOrder(i) != prod2.getOrder(i)) { return false; } } return true; }
/** * Set the parameters for range partition. * * @param ordering Order of the partitioned fields */ public void setRangePartitioned(Ordering ordering) { if (ordering == null) { throw new NullPointerException(); } this.partitioning = PartitioningProperty.RANGE_PARTITIONED; this.ordering = ordering; this.partitioningFields = ordering.getInvolvedIndexes(); }
public Ordering clone() { Ordering newOrdering = new Ordering(); newOrdering.indexes = this.indexes; newOrdering.types.addAll(this.types); newOrdering.orders.addAll(this.orders); return newOrdering; }
/** * @param index * @param type * @param order */ public Ordering(int index, Class<? extends Comparable<?>> type, Order order) { appendOrdering(index, type, order); }
/** * Creates a new ordering the represents an ordering on a prefix of the fields. If the * exclusive index up to which to create the ordering is <code>0</code>, then there is * no resulting ordering and this method return <code>null</code>. * * @param exclusiveIndex The index (exclusive) up to which to create the ordering. * @return The new ordering on the prefix of the fields, or <code>null</code>, if the prefix is empty. */ public Ordering createNewOrderingUpToIndex(int exclusiveIndex) { if (exclusiveIndex == 0) { return null; } final Ordering newOrdering = new Ordering(); for (int i = 0; i < exclusiveIndex; i++) { newOrdering.appendOrdering(this.indexes.get(i), this.types.get(i), this.orders.get(i)); } return newOrdering; }
this.splitOrdering = new Ordering(); for (int okey : splitOrdering.getFieldPositions()) { if (key == okey) { throw new InvalidProgramException("Duplicate field in field expression " + keyExp); this.splitOrdering.appendOrdering(key, null, orders[i]);