for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) { aggregators.put(a.getName(), a.getAggregator()); String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName(); ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
/** * Registers an {@link Aggregator} for the iteration together with a {@link ConvergenceCriterion}. For a general description * of aggregators, see {@link #registerAggregator(String, Aggregator)} and {@link Aggregator}. * At the end of each iteration, the convergence criterion takes the aggregator's global aggregate value and decides whether * the iteration should terminate. A typical use case is to have an aggregator that sums up the total error of change * in an iteration step and have to have a convergence criterion that signals termination as soon as the aggregate value * is below a certain threshold. * * @param name The name under which the aggregator is registered. * @param aggregator The aggregator class. * @param convergenceCheck The convergence criterion. * * @return The DeltaIteration itself, to allow chaining function calls. */ @PublicEvolving public <X extends Value> DeltaIteration<ST, WT> registerAggregationConvergenceCriterion( String name, Aggregator<X> aggregator, ConvergenceCriterion<X> convergenceCheck) { this.aggregators.registerAggregationConvergenceCriterion(name, aggregator, convergenceCheck); return this; }
/** * Registers an {@link Aggregator} for the iteration. Aggregators can be used to maintain simple statistics during the * iteration, such as number of elements processed. The aggregators compute global aggregates: After each iteration step, * the values are globally aggregated to produce one aggregate that represents statistics across all parallel instances. * The value of an aggregator can be accessed in the next iteration. * * <p>Aggregators can be accessed inside a function via the * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getIterationRuntimeContext()} method. * * @param name The name under which the aggregator is registered. * @param aggregator The aggregator class. * * @return The DeltaIteration itself, to allow chaining function calls. */ @PublicEvolving public DeltaIteration<ST, WT> registerAggregator(String name, Aggregator<?> aggregator) { this.aggregators.registerAggregator(name, aggregator); return this; }
this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO(); else this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();
assertEquals(aggregatorName, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) { @SuppressWarnings("unchecked") BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd; IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead(); BulkIterationBase<T> iterationOperator = new BulkIterationBase<>(new UnaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration"); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } translated.put(iterationHead, iterationOperator.getPartialSolution()); Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution()); iterationOperator.setNextPartialSolution(translatedBody); iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations()); iterationOperator.setInput(translate(iterationHead.getInput())); iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); if (iterationEnd.getTerminationCriterion() != null) { iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion())); } return iterationOperator; }
this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO(); else this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();
private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) { @SuppressWarnings("unchecked") DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd; DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead(); String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName(); DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()), iterationEnd.getKeyPositions(), name); iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations()); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet(); DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset(); translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet()); translated.put(worksetPlaceHolder, iterationOperator.getWorkset()); Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet()); Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset()); iterationOperator.setNextWorkset(translatedWorkset); iterationOperator.setSolutionSetDelta(translatedSolutionSet); iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet())); iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset())); // register all aggregators iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged()); return iterationOperator; }
for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) { aggregators.put(a.getName(), a.getAggregator()); String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName(); ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
/** * Registers an {@link Aggregator} for the iteration together with a {@link ConvergenceCriterion}. For a general description * of aggregators, see {@link #registerAggregator(String, Aggregator)} and {@link Aggregator}. * At the end of each iteration, the convergence criterion takes the aggregator's global aggregate value and decided whether * the iteration should terminate. A typical use case is to have an aggregator that sums up the total error of change * in an iteration step and have to have a convergence criterion that signals termination as soon as the aggregate value * is below a certain threshold. * * @param name The name under which the aggregator is registered. * @param aggregator The aggregator class. * @param convergenceCheck The convergence criterion. * * @return The IterativeDataSet itself, to allow chaining function calls. */ @PublicEvolving public <X extends Value> IterativeDataSet<T> registerAggregationConvergenceCriterion( String name, Aggregator<X> aggregator, ConvergenceCriterion<X> convergenceCheck) { this.aggregators.registerAggregationConvergenceCriterion(name, aggregator, convergenceCheck); return this; }
this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO(); else this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();
/** * Registers an {@link Aggregator} for the iteration. Aggregators can be used to maintain simple statistics during the * iteration, such as number of elements processed. The aggregators compute global aggregates: After each iteration step, * the values are globally aggregated to produce one aggregate that represents statistics across all parallel instances. * The value of an aggregator can be accessed in the next iteration. * * <p>Aggregators can be accessed inside a function via the * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getIterationRuntimeContext()} method. * * @param name The name under which the aggregator is registered. * @param aggregator The aggregator class. * * @return The IterativeDataSet itself, to allow chaining function calls. */ @PublicEvolving public IterativeDataSet<T> registerAggregator(String name, Aggregator<?> aggregator) { this.aggregators.registerAggregator(name, aggregator); return this; }
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) { @SuppressWarnings("unchecked") BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd; IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead(); BulkIterationBase<T> iterationOperator = new BulkIterationBase<>(new UnaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration"); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } translated.put(iterationHead, iterationOperator.getPartialSolution()); Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution()); iterationOperator.setNextPartialSolution(translatedBody); iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations()); iterationOperator.setInput(translate(iterationHead.getInput())); iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); if (iterationEnd.getTerminationCriterion() != null) { iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion())); } return iterationOperator; }
Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators(); String convAggName = aggs.getConvergenceCriterionAggregatorName(); ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
/** * @param criterion */ public <X> void setTerminationCriterion(Operator<X> criterion) { TypeInformation<X> type = criterion.getOperatorInfo().getOutputType(); FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>> mapper = new FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>>( new TerminationCriterionMapper<X>(), new UnaryOperatorInformation<X, X>(type, type), "Termination Criterion Aggregation Wrapper"); mapper.setInput(criterion); this.terminationCriterion = mapper; this.getAggregators().registerAggregationConvergenceCriterion(TERMINATION_CRITERION_AGGREGATOR_NAME, new TerminationCriterionAggregator(), new TerminationCriterionAggregationConvergence()); }
this.globalELBO = ((ConvergenceELBO)loop.getAggregators().getConvergenceCriterion()).getELBO(); else this.globalELBO = ((ConvergenceELBObyTime)loop.getAggregators().getConvergenceCriterion()).getELBO();
/** * Registers an {@link Aggregator} for the iteration. Aggregators can be used to maintain simple statistics during the * iteration, such as number of elements processed. The aggregators compute global aggregates: After each iteration step, * the values are globally aggregated to produce one aggregate that represents statistics across all parallel instances. * The value of an aggregator can be accessed in the next iteration. * * <p>Aggregators can be accessed inside a function via the * {@link org.apache.flink.api.common.functions.AbstractRichFunction#getIterationRuntimeContext()} method. * * @param name The name under which the aggregator is registered. * @param aggregator The aggregator class. * * @return The IterativeDataSet itself, to allow chaining function calls. */ @PublicEvolving public IterativeDataSet<T> registerAggregator(String name, Aggregator<?> aggregator) { this.aggregators.registerAggregator(name, aggregator); return this; }
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) { @SuppressWarnings("unchecked") BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd; IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead(); BulkIterationBase<T> iterationOperator = new BulkIterationBase<>(new UnaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration"); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } translated.put(iterationHead, iterationOperator.getPartialSolution()); Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution()); iterationOperator.setNextPartialSolution(translatedBody); iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations()); iterationOperator.setInput(translate(iterationHead.getInput())); iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); if (iterationEnd.getTerminationCriterion() != null) { iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion())); } return iterationOperator; }
Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators(); syncConfig.addIterationAggregators(allAggregators); String convAggName = aggs.getConvergenceCriterionAggregatorName(); ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
iteration.getAggregators().registerAggregationConvergenceCriterion( AGGREGATOR_NAME, new PageRankStatsAggregator(),