@SuppressWarnings("unchecked") private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception { Operator<?> solutionInput = iteration.getInitialSolutionSet(); Operator<?> worksetInput = iteration.getInitialWorkset(); if (solutionInput == null) { throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set."); throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset."); if (iteration.getSolutionSetDelta() == null) { throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed)."); if (iteration.getNextWorkset() == null) { throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed)."); iteration.getSolutionSetDelta().accept(dynCollector); iteration.getNextWorkset().accept(dynCollector); BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo(); TypeInformation<?> solutionType = operatorInfo.getFirstInputType(); int[] keyColumns = iteration.getSolutionSetKeyFields(); boolean[] inputOrderings = new boolean[keyColumns.length]; TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig); for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) { aggregators.put(a.getName(), a.getAggregator()); String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName(); ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
/** * Creates a new node with a single input for the optimizer plan. * * @param iteration The iteration operator that the node represents. */ public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) { super(iteration); final int[] ssKeys = iteration.getSolutionSetKeyFields(); if (ssKeys == null || ssKeys.length == 0) { throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set."); } this.solutionSetKeyFields = new FieldList(ssKeys); this.partitionedProperties = new GlobalProperties(); this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields); int weight = iteration.getMaximumNumberOfIterations() > 0 ? iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT; if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) { weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; } this.costWeight = weight; this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields)); }
private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) { @SuppressWarnings("unchecked") DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd; DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead(); String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName(); DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()), iterationEnd.getKeyPositions(), name); iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations()); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet(); DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset(); translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet()); translated.put(worksetPlaceHolder, iterationOperator.getWorkset()); Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet()); Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset()); iterationOperator.setNextWorkset(translatedWorkset); iterationOperator.setSolutionSetDelta(translatedSolutionSet); iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet())); iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset())); // register all aggregators iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged()); return iterationOperator; }
iter.getNextWorkset().accept(wsf); if (!wsf.hasFoundWorkset()) { throw new CompilerException("In the given program, the next workset does not depend on the workset. " + iter.getSolutionSetDelta().accept(recursiveCreator); final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset()); iter.getNextWorkset().accept(recursiveCreator); SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet()); solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode); final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset()); final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
assertEquals(numIterations, iteration.getMaximumNumberOfIterations()); assertArrayEquals(iterationKeys, iteration.getSolutionSetKeyFields()); assertEquals(iterationParallelism, iteration.getParallelism()); assertEquals(iterationName, iteration.getName()); MapOperatorBase<?, ?, ?> nextWorksetMapper = (MapOperatorBase<?, ?, ?>) iteration.getNextWorkset(); InnerJoinOperatorBase<?, ?, ?, ?> solutionSetJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) iteration.getSolutionSetDelta(); InnerJoinOperatorBase<?, ?, ?, ?> worksetSelfJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) solutionSetJoin.getFirstInput(); MapOperatorBase<?, ?, ?> worksetMapper = (MapOperatorBase<?, ?, ?>) worksetSelfJoin.getFirstInput(); assertEquals(aggregatorName, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations."); AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators(); Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
@Override public boolean preVisit(Operator<?> visitable) { if (this.seen.add(visitable)) { // add to the map final String name = visitable.getName(); List<Operator<?>> list = this.map.get(name); if (list == null) { list = new ArrayList<Operator<?>>(2); this.map.put(name, list); } list.add(visitable); // recurse into bulk iterations if (visitable instanceof BulkIterationBase) { ((BulkIterationBase) visitable).getNextPartialSolution().accept(this); } else if (visitable instanceof DeltaIterationBase) { ((DeltaIterationBase) visitable).getSolutionSetDelta().accept(this); ((DeltaIterationBase) visitable).getNextWorkset().accept(this); } return true; } else { return false; } }
iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType())); iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType())); iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
iter.getNextWorkset().accept(wsf); if (!wsf.hasFoundWorkset()) { throw new CompilerException("In the given program, the next workset does not depend on the workset. " + iter.getSolutionSetDelta().accept(recursiveCreator); final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset()); iter.getNextWorkset().accept(recursiveCreator); SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet()); solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode); final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset()); final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) { @SuppressWarnings("unchecked") DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd; DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead(); String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName(); DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()), iterationEnd.getKeyPositions(), name); iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations()); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet(); DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset(); translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet()); translated.put(worksetPlaceHolder, iterationOperator.getWorkset()); Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet()); Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset()); iterationOperator.setNextWorkset(translatedWorkset); iterationOperator.setSolutionSetDelta(translatedSolutionSet); iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet())); iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset())); // register all aggregators iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged()); return iterationOperator; }
final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations."); AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators(); Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
/** * Creates a new node with a single input for the optimizer plan. * * @param iteration The iteration operator that the node represents. */ public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) { super(iteration); final int[] ssKeys = iteration.getSolutionSetKeyFields(); if (ssKeys == null || ssKeys.length == 0) { throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set."); } this.solutionSetKeyFields = new FieldList(ssKeys); this.partitionedProperties = new GlobalProperties(); this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields); int weight = iteration.getMaximumNumberOfIterations() > 0 ? iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT; if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) { weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; } this.costWeight = weight; this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields)); }
iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType())); iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType())); iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));
@SuppressWarnings("unchecked") private <T> List<T> executeDeltaIteration(DeltaIterationBase<?, ?> iteration) throws Exception { Operator<?> solutionInput = iteration.getInitialSolutionSet(); Operator<?> worksetInput = iteration.getInitialWorkset(); if (solutionInput == null) { throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial solution set."); throw new InvalidProgramException("The delta iteration " + iteration.getName() + " has no initial workset."); if (iteration.getSolutionSetDelta() == null) { throw new InvalidProgramException("The iteration " + iteration.getName() + " has no solution set delta defined (is not closed)."); if (iteration.getNextWorkset() == null) { throw new InvalidProgramException("The iteration " + iteration.getName() + " has no workset defined (is not closed)."); iteration.getSolutionSetDelta().accept(dynCollector); iteration.getNextWorkset().accept(dynCollector); BinaryOperatorInformation<?, ?, ?> operatorInfo = iteration.getOperatorInfo(); TypeInformation<?> solutionType = operatorInfo.getFirstInputType(); int[] keyColumns = iteration.getSolutionSetKeyFields(); boolean[] inputOrderings = new boolean[keyColumns.length]; TypeComparator<T> inputComparator = ((CompositeType<T>) solutionType).createComparator(keyColumns, inputOrderings, 0, executionConfig); for (AggregatorWithName<?> a : iteration.getAggregators().getAllRegisteredAggregators()) { aggregators.put(a.getName(), a.getAggregator()); String convCriterionAggName = iteration.getAggregators().getConvergenceCriterionAggregatorName(); ConvergenceCriterion<Value> convCriterion = (ConvergenceCriterion<Value>) iteration.getAggregators().getConvergenceCriterion();
iter.getNextWorkset().accept(wsf); if (!wsf.hasFoundWorkset()) { throw new CompilerException("In the given program, the next workset does not depend on the workset. " + iter.getSolutionSetDelta().accept(recursiveCreator); final WorksetNode worksetNode = (WorksetNode) recursiveCreator.con2node.get(iter.getWorkset()); iter.getNextWorkset().accept(recursiveCreator); SolutionSetNode solutionSetNode = (SolutionSetNode) recursiveCreator.con2node.get(iter.getSolutionSet()); solutionSetNode = new SolutionSetNode((DeltaIterationBase.SolutionSetPlaceHolder<?>) iter.getSolutionSet(), iterNode); final OptimizerNode nextWorksetNode = recursiveCreator.con2node.get(iter.getNextWorkset()); final OptimizerNode solutionSetDeltaNode = recursiveCreator.con2node.get(iter.getSolutionSetDelta());
private <D, W> DeltaIterationBase<D, W> translateDeltaIteration(DeltaIterationResultSet<?, ?> untypedIterationEnd) { @SuppressWarnings("unchecked") DeltaIterationResultSet<D, W> iterationEnd = (DeltaIterationResultSet<D, W>) untypedIterationEnd; DeltaIteration<D, W> iterationHead = iterationEnd.getIterationHead(); String name = iterationHead.getName() == null ? "Unnamed Delta Iteration" : iterationHead.getName(); DeltaIterationBase<D, W> iterationOperator = new DeltaIterationBase<>(new BinaryOperatorInformation<>(iterationEnd.getType(), iterationEnd.getWorksetType(), iterationEnd.getType()), iterationEnd.getKeyPositions(), name); iterationOperator.setMaximumNumberOfIterations(iterationEnd.getMaxIterations()); if (iterationHead.getParallelism() > 0) { iterationOperator.setParallelism(iterationHead.getParallelism()); } DeltaIteration.SolutionSetPlaceHolder<D> solutionSetPlaceHolder = iterationHead.getSolutionSet(); DeltaIteration.WorksetPlaceHolder<W> worksetPlaceHolder = iterationHead.getWorkset(); translated.put(solutionSetPlaceHolder, iterationOperator.getSolutionSet()); translated.put(worksetPlaceHolder, iterationOperator.getWorkset()); Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet()); Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset()); iterationOperator.setNextWorkset(translatedWorkset); iterationOperator.setSolutionSetDelta(translatedSolutionSet); iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet())); iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset())); // register all aggregators iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); iterationOperator.setSolutionSetUnManaged(iterationHead.isSolutionSetUnManaged()); return iterationOperator; }
final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations."); AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators(); Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
/** * Creates a new node with a single input for the optimizer plan. * * @param iteration The iteration operator that the node represents. */ public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) { super(iteration); final int[] ssKeys = iteration.getSolutionSetKeyFields(); if (ssKeys == null || ssKeys.length == 0) { throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set."); } this.solutionSetKeyFields = new FieldList(ssKeys); this.partitionedProperties = new GlobalProperties(); this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields); int weight = iteration.getMaximumNumberOfIterations() > 0 ? iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT; if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) { weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; } this.costWeight = weight; this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields)); }
iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType())); iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType())); iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null)));