@Override public Plan getPlan(String... args) throws IllegalArgumentException { // parse program parameters int numSubtasks = (args.length > 0 ? Integer.parseInt(args[0]) : 1); String recordsPath = (args.length > 1 ? args[1] : ""); String output = (args.length > 2 ? args[2] : ""); FileDataSource source = new FileDataSource(CsvInputFormat.class, recordsPath); source.setDegreeOfParallelism(numSubtasks); CsvInputFormat.configureRecordFormat(source) .recordDelimiter('\n') .fieldDelimiter('|') .field(IntValue.class, 0); FileDataSink sink = new FileDataSink(CsvOutputFormat.class, output); sink.setDegreeOfParallelism(numSubtasks); CsvOutputFormat.configureRecordFormat(sink) .recordDelimiter('\n') .fieldDelimiter('|') .lenient(true) .field(IntValue.class, 0); sink.setGlobalOrder(new Ordering(0, IntValue.class, Order.ASCENDING), new UniformIntegerDistribution(Integer.MIN_VALUE, Integer.MAX_VALUE)); sink.setInput(source); return new Plan(sink); }
.recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) // order id .field(IntValue.class, 7) // ship prio .field(StringValue.class, 2, 2) // order status .field(StringValue.class, 4, 10) // order date .field(StringValue.class, 5, 8); // order prio .recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) // order id .field(DoubleValue.class, 5); // extended price
.recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) // order id .field(IntValue.class, 7) // ship prio .field(StringValue.class, 2, 2) // order status .field(StringValue.class, 4, 10) // order date .field(StringValue.class, 5, 8); // order prio .recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) // order id .field(IntValue.class, 7) // ship prio .field(StringValue.class, 2, 2) // order status .field(StringValue.class, 4, 10) // order date .field(StringValue.class, 5, 8); // order prio .recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) .field(DoubleValue.class, 5); .recordDelimiter('\n') .fieldDelimiter('|') .field(LongValue.class, 0) .field(IntValue.class, 1) .field(DoubleValue.class, 2); .recordDelimiter('\n') .fieldDelimiter('|')
@Override protected Plan getTestJob() { // Sc1 generates M parameters a,b,c for second degree polynomials P(x) = ax^2 + bx + c identified by id FileDataSource sc1 = new FileDataSource(new CsvInputFormat(), sc1Path); CsvInputFormat.configureRecordFormat(sc1).fieldDelimiter(' ').field(StringValue.class, 0).field(IntValue.class, 1) .field(IntValue.class, 2).field(IntValue.class, 3); // Sc2 generates N x values to be evaluated with the polynomial identified by id FileDataSource sc2 = new FileDataSource(new CsvInputFormat(), sc2Path); CsvInputFormat.configureRecordFormat(sc2).fieldDelimiter(' ').field(StringValue.class, 0).field(IntValue.class, 1); // Sc3 generates N y values to be evaluated with the polynomial identified by id FileDataSource sc3 = new FileDataSource(new CsvInputFormat(), sc3Path); CsvInputFormat.configureRecordFormat(sc3).fieldDelimiter(' ').field(StringValue.class, 0).field(IntValue.class, 1); // Jn1 matches x and y values on id and emits (id, x, y) triples JoinOperator jn1 = JoinOperator.builder(Jn1.class, StringValue.class, 0, 0).input1(sc2).input2(sc3).build(); // Jn2 matches polynomial and arguments by id, computes p = min(P(x),P(y)) and emits (id, p) tuples JoinOperator jn2 = JoinOperator.builder(Jn2.class, StringValue.class, 0, 0).input1(jn1).input2(sc1).build(); // Mp1 selects (id, x, y) triples where x = y and broadcasts z (=x=y) to Mp2 MapOperator mp1 = MapOperator.builder(Mp1.class).input(jn1).build(); // Mp2 filters out all p values which can be divided by z MapOperator mp2 = MapOperator.builder(Mp2.class).setBroadcastVariable("z", mp1).input(jn2).build(); FileDataSink output = new FileDataSink(new ContractITCaseOutputFormat(), resultPath); output.setDegreeOfParallelism(1); output.setInput(mp2); return new Plan(output); }