public Ordering clone() { final Ordering newOrdering = new Ordering(); newOrdering.indexes.addAll(this.indexes); newOrdering.types.addAll(this.types); newOrdering.orders.addAll(this.orders); return newOrdering; }
public Ordering clone() { final Ordering newOrdering = new Ordering(); newOrdering.indexes.addAll(this.indexes); newOrdering.types.addAll(this.types); newOrdering.orders.addAll(this.orders); return newOrdering; }
/** * Creates a new ordering the represents an ordering on a prefix of the fields. If the * exclusive index up to which to create the ordering is <code>0</code>, then there is * no resulting ordering and this method return <code>null</code>. * * @param exclusiveIndex The index (exclusive) up to which to create the ordering. * @return The new ordering on the prefix of the fields, or <code>null</code>, if the prefix is empty. */ public Ordering createNewOrderingUpToIndex(int exclusiveIndex) { if (exclusiveIndex == 0) { return null; } final Ordering newOrdering = new Ordering(); for (int i = 0; i < exclusiveIndex; i++) { newOrdering.appendOrdering(this.indexes.get(i), this.types.get(i), this.orders.get(i)); } return newOrdering; }
/** * Creates a new ordering the represents an ordering on a prefix of the fields. If the * exclusive index up to which to create the ordering is <code>0</code>, then there is * no resulting ordering and this method return <code>null</code>. * * @param exclusiveIndex The index (exclusive) up to which to create the ordering. * @return The new ordering on the prefix of the fields, or <code>null</code>, if the prefix is empty. */ public Ordering createNewOrderingUpToIndex(int exclusiveIndex) { if (exclusiveIndex == 0) { return null; } final Ordering newOrdering = new Ordering(); for (int i = 0; i < exclusiveIndex; i++) { newOrdering.appendOrdering(this.indexes.get(i), this.types.get(i), this.orders.get(i)); } return newOrdering; }
public static final Ordering createOrdering(FieldList fields) { final Ordering o = new Ordering(); for (int i = 0; i < fields.size(); i++) { o.appendOrdering(fields.get(i), null, Order.ANY); } return o; }
public static final Ordering createOrdering(FieldList fields, boolean[] directions) { final Ordering o = new Ordering(); for (int i = 0; i < fields.size(); i++) { o.appendOrdering(fields.get(i), null, directions == null || directions[i] ? Order.ASCENDING : Order.DESCENDING); } return o; }
public static final Ordering createOrdering(FieldList fields) { final Ordering o = new Ordering(); for (int i = 0; i < fields.size(); i++) { o.appendOrdering(fields.get(i), null, Order.ANY); } return o; }
public static final Ordering createOrdering(FieldList fields, boolean[] directions) { final Ordering o = new Ordering(); for (int i = 0; i < fields.size(); i++) { o.appendOrdering(fields.get(i), null, directions == null || directions[i] ? Order.ASCENDING : Order.DESCENDING); } return o; }
public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } }
public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } }
public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } }
public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } }
Order[] sortOrders = sortedGrouper.getGroupSortOrders(); Ordering o = new Ordering(); for(int i=0; i < sortKeyPositions.length; i++) { o.appendOrdering(sortKeyPositions[i], null, sortOrders[i]);
@Override public Plan getPlan(String... args) throws IllegalArgumentException { // parse job parameters final int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); final String input = (args.length > 1 ? args[1] : ""); final String output = (args.length > 2 ? args[2] : ""); // This task will read the input data and generate the key/value pairs final FileDataSource source = new FileDataSource(new TeraInputFormat(), input, "Data Source"); source.setDegreeOfParallelism(numSubTasks); // This task writes the sorted data back to disk final FileDataSink sink = new FileDataSink(new TeraOutputFormat(), output, "Data Sink"); sink.setDegreeOfParallelism(numSubTasks); sink.setGlobalOrder(new Ordering(0, TeraKey.class, Order.ASCENDING), new TeraDistribution()); sink.setInput(source); return new Plan(sink, "TeraSort"); } }
@Override public Plan getPlan(String... args) throws IllegalArgumentException { // parse program parameters int numSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); String recordsPath = (args.length > 1 ? args[1] : ""); String output = (args.length > 2 ? args[2] : ""); FileDataSource source = new FileDataSource(CsvInputFormat.class, recordsPath); source.setDegreeOfParallelism(numSubtasks); CsvInputFormat.configureRecordFormat(source) .recordDelimiter('\n') .fieldDelimiter('|') .field(IntValue.class, 0); FileDataSink sink = new FileDataSink(CsvOutputFormat.class, output); sink.setDegreeOfParallelism(numSubtasks); CsvOutputFormat.configureRecordFormat(sink) .recordDelimiter('\n') .fieldDelimiter('|') .lenient(true) .field(IntValue.class, 0); sink.setGlobalOrder(new Ordering(0, IntValue.class, Order.ASCENDING), new UniformIntegerDistribution(Integer.MIN_VALUE, Integer.MAX_VALUE)); sink.setInput(source); return new Plan(sink); }
@Override public Plan getPlan(String... args) { // parse job parameters int numSubTasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); String dataInput = (args.length > 1 ? args[1] : ""); String output = (args.length > 2 ? args[2] : ""); @SuppressWarnings("unchecked") CsvInputFormat format = new CsvInputFormat(' ', IntValue.class, IntValue.class); FileDataSource input = new FileDataSource(format, dataInput, "Input"); // create the reduce contract and sets the key to the first field ReduceOperator sorter = ReduceOperator.builder(new IdentityReducer(), IntValue.class, 0) .input(input) .name("Reducer") .build(); // sets the group sorting to the second field sorter.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING)); // create and configure the output format FileDataSink out = new FileDataSink(new CsvOutputFormat(), output, sorter, "Sorted Output"); CsvOutputFormat.configureRecordFormat(out) .recordDelimiter('\n') .fieldDelimiter(' ') .field(IntValue.class, 0) .field(IntValue.class, 1); Plan plan = new Plan(out, "SecondarySort Example"); plan.setDefaultParallelism(numSubTasks); return plan; }
new Ordering(0, IntValue.class, Order.DESCENDING) .appendOrdering(1, IntValue.class, Order.ASCENDING) .appendOrdering(2, IntValue.class, Order.DESCENDING),
@Override protected Plan getTestJob() { int dop = this.config.getInteger("GroupOrderTest#NumSubtasks", 1); @SuppressWarnings("unchecked") CsvInputFormat format = new CsvInputFormat(',', IntValue.class, IntValue.class); FileDataSource source = new FileDataSource(format, this.textPath, "Source"); ReduceOperator reducer = ReduceOperator.builder(CheckingReducer.class) .keyField(IntValue.class, 0) .input(source) .name("Ordered Reducer") .build(); reducer.setGroupOrder(new Ordering(1, IntValue.class, Order.ASCENDING)); FileDataSink sink = new FileDataSink(CsvOutputFormat.class, this.resultPath, reducer, "Sink"); CsvOutputFormat.configureRecordFormat(sink) .recordDelimiter('\n') .fieldDelimiter(',') .field(IntValue.class, 0) .field(IntValue.class, 1); Plan p = new Plan(sink); p.setDefaultParallelism(dop); return p; }