@Override public Iterator<Row> call(Tuple2<Row, Tuple2<Iterable<Row>, Iterable<Row>>> keyedRecords) throws Exception { long startTime = System.nanoTime(); if (planner == null) { planner = (RandomPlanner)PlannerFactory.create(config, true); if (planner instanceof UsesAccumulators) { ((UsesAccumulators)planner).receiveAccumulators(accumulators); } } Row key = keyedRecords._1(); List<Row> arrivingRecords = Lists.newArrayList(keyedRecords._2()._1()); List<Row> existingRecords = Lists.newArrayList(keyedRecords._2()._2()); Iterable<Row> plannedForKey = planner.planMutationsForKey(key, arrivingRecords, existingRecords); long endTime = System.nanoTime(); accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_PLANNING).add( (endTime - startTime) / 1000.0 / 1000.0 / 1000.0); return plannedForKey.iterator(); } };
private void writeOutput() { Config plannerConfig = config.getConfig(PLANNER_TYPE); Planner planner = getPlanner(true); validatePlannerOutputCompatibility(planner, getOutput(false)); // Plan the mutations, and then apply them to the output, based on the type of planner used if (planner instanceof RandomPlanner) { RandomPlanner randomPlanner = (RandomPlanner)planner; List<String> keyFieldNames = randomPlanner.getKeyFieldNames(); Config outputConfig = config.getConfig(OUTPUT_TYPE); JavaRDD<Row> planned = planMutationsByKey(data, keyFieldNames, plannerConfig, outputConfig); applyMutations(planned, outputConfig); } else if (planner instanceof BulkPlanner) { BulkPlanner bulkPlanner = (BulkPlanner)planner; List<Tuple2<MutationType, Dataset<Row>>> planned = bulkPlanner.planMutationsForSet(data); BulkOutput bulkOutput = (BulkOutput)getOutput(true); bulkOutput.applyBulkMutations(planned); } else { throw new RuntimeException("Unexpected planner class: " + planner.getClass().getName()); } }