return strategy.name();
chaining = ds.getPushChainDriverClass() != null && !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways vertex = null; config = new TaskConfig(new Configuration()); this.chainedTasks.put(node, new TaskInChain(node, ds.getPushChainDriverClass(), config, taskName)); } else { config.setDriver(ds.getDriverClass()); for (int i = 0; i < ds.getNumRequiredComparators(); i++) { config.setDriverComparator(node.getComparator(i), i);
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders) { super(template, nodeName, driverStrategy); this.input = input; this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()]; this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()]; this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][]; if(driverStrategy.getNumRequiredComparators() > 0) { this.driverKeys[0] = driverKeyFields; this.driverSortOrders[0] = driverSortOrders; } if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input.setReplicationFactor(getParallelism()); } final PlanNode predNode = input.getSource(); if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) { if (this.branchPlan == null) { this.branchPlan = new HashMap<OptimizerNode, PlanNode>(); } this.branchPlan.putAll(predNode.branchPlan); } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
public int getMemoryConsumerWeight() { return this.driverStrategy.isMaterializing() ? 1 : 0; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) { PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
JobVertex headVertex = this.iterations.get(wsNode).getHeadTask(); TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration()); int inputIndex = headConfig.getDriverStrategy().getNumInputs(); headConfig.setIterationHeadSolutionSetInputIndex(inputIndex); translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
public void setDriverStrategy(DriverStrategy strategy) { this.config.setInteger(DRIVER_STRATEGY, strategy.ordinal()); }
/** * Sets the key field information for the specified driver comparator. * * @param keys The key field indexes for the specified driver comparator. * @param sortOrder The key sort order for the specified driver comparator. * @param id The ID of the driver comparator. */ public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) { if(id < 0 || id >= driverKeys.length) { throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only " +super.getDriverStrategy().getNumRequiredComparators()+" comparators."); } this.driverKeys[id] = keys; this.driverSortOrders[id] = sortOrder; }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
public int getMemoryConsumerWeight() { return this.driverStrategy.isMaterializing() ? 1 : 0; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
source.getDriverStrategy().equals(DriverStrategy.SORTED_GROUP_COMBINE)) { PlanUnwrappingReduceGroupOperator<?, ?, ?> groupNode = (PlanUnwrappingReduceGroupOperator<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType();
JobVertex headVertex = this.iterations.get(wsNode).getHeadTask(); TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration()); int inputIndex = headConfig.getDriverStrategy().getNumInputs(); headConfig.setIterationHeadSolutionSetInputIndex(inputIndex); translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false);
public void setDriverStrategy(DriverStrategy strategy) { this.config.setInteger(DRIVER_STRATEGY, strategy.ordinal()); }
locString = p.getDriverStrategy().name(); break;
chaining = ds.getPushChainDriverClass() != null && !(pred instanceof NAryUnionPlanNode) && // first op after union is stand-alone, because union is merged !(pred instanceof BulkPartialSolutionPlanNode) && // partial solution merges anyways vertex = null; config = new TaskConfig(new Configuration()); this.chainedTasks.put(node, new TaskInChain(node, ds.getPushChainDriverClass(), config, taskName)); } else { config.setDriver(ds.getDriverClass()); for (int i = 0; i < ds.getNumRequiredComparators(); i++) { config.setDriverComparator(node.getComparator(i), i);
boolean damOnAllRightPaths = true; if (strategy.firstDam() == DamBehavior.FULL_DAM || in1.getLocalStrategy().dams() || in1.getTempMode().breaksPipeline()) { someDamOnLeftPaths = true; } else { if (strategy.secondDam() == DamBehavior.FULL_DAM || in2.getLocalStrategy().dams() || in2.getTempMode().breaksPipeline()) { someDamOnRightPaths = true; } else {