/** * Instantiates a new SparkMDoubleAccumulator. * * @param name the name of the accumulator */ public SparkMDoubleAccumulator(String name) { super(new DoubleAccumulator()); this.name = name; }
@Override public Row call(Row arrived) throws Exception { long startTime = System.nanoTime(); if (schema == null) { schema = RowUtils.subsetSchema(arrived.schema(), keyFieldNames); } Row key = RowUtils.subsetRow(arrived, schema); long endTime = System.nanoTime(); accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_EXTRACTING_KEYS).add( (endTime - startTime) / 1000.0 / 1000.0 / 1000.0); return key; } }
@Test public void testRequestMany() { AccumulatorRequest request1 = new AccumulatorRequest("hello", Long.class); AccumulatorRequest request2 = new AccumulatorRequest("world", Double.class); Accumulators accumulators = new Accumulators(Sets.newHashSet(request1, request2)); LongAccumulator accumulator1 = accumulators.getLongAccumulators().get("hello"); assertEquals(accumulator1.name().get(), "hello"); DoubleAccumulator accumulator2 = accumulators.getDoubleAccumulators().get("world"); assertEquals(accumulator2.name().get(), "world"); }
@Override public void call(Iterator<Row> plannedIterator) throws Exception { long startTime = System.nanoTime(); if (output == null) { output = (RandomOutput)OutputFactory.create(config, true); if (output instanceof UsesAccumulators) { ((UsesAccumulators)output).receiveAccumulators(accumulators); } } List<Row> planned = Lists.newArrayList(plannedIterator); output.applyRandomMutations(planned); long endTime = System.nanoTime(); accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_APPLYING).add( (endTime - startTime) / 1000.0 / 1000.0 / 1000.0); } }
@Override public Iterable<Row> getExistingForFilters(Iterable<Row> filters) throws Exception { List<Row> existingForFilters = Lists.newArrayList(); if (!filters.iterator().hasNext()) { return existingForFilters; } KuduTable table = getConnection().getTable(config.getString(TABLE_CONFIG_NAME)); KuduScanner scanner = scannerForFilters(filters, table); long startTime = System.nanoTime(); while (scanner.hasMoreRows()) { for (RowResult rowResult : scanner.nextRows()) { Row existing = resultAsRow(rowResult, table); existingForFilters.add(existing); } } long endTime = System.nanoTime(); if (hasAccumulators()) { accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_SCANNING).add((endTime - startTime) / 1000.0 / 1000.0 / 1000.0); } return existingForFilters; }
accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_EXISTING).add( (endTime - startTime) / 1000.0 / 1000.0 / 1000.0);
@Override public Iterator<Row> call(Tuple2<Row, Tuple2<Iterable<Row>, Iterable<Row>>> keyedRecords) throws Exception { long startTime = System.nanoTime(); if (planner == null) { planner = (RandomPlanner)PlannerFactory.create(config, true); if (planner instanceof UsesAccumulators) { ((UsesAccumulators)planner).receiveAccumulators(accumulators); } } Row key = keyedRecords._1(); List<Row> arrivingRecords = Lists.newArrayList(keyedRecords._2()._1()); List<Row> existingRecords = Lists.newArrayList(keyedRecords._2()._2()); Iterable<Row> plannedForKey = planner.planMutationsForKey(key, arrivingRecords, existingRecords); long endTime = System.nanoTime(); accumulators.getDoubleAccumulators().get(ACCUMULATOR_SECONDS_PLANNING).add( (endTime - startTime) / 1000.0 / 1000.0 / 1000.0); return plannedForKey.iterator(); } };