private void finalizeWorksetIteration(IterationDescriptor descr) { final WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) descr.getIterationNode(); final JobVertex headVertex = descr.getHeadTask(); final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration()); final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig(); final int numStepFunctionOuts = headConfig.getNumOutputs(); final int numFinalOuts = headFinalOutputConfig.getNumOutputs(); headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig); headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts); final double relativeMemory = iterNode.getRelativeMemoryPerSubTask(); if (relativeMemory <= 0) { headConfig.setIsWorksetIteration(); headConfig.setRelativeBackChannelMemory(relativeMemory / 2); headConfig.setRelativeSolutionSetMemory(relativeMemory / 2); headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer()); headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator()); syncConfig = new TaskConfig(sync.getConfiguration()); syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism()); syncConfig.setNumberOfIterations(maxNumIterations); worksetTailConfig = taskInChain.getTaskConfig(); } else { worksetTailConfig = new TaskConfig(nextWorksetVertex.getConfiguration());
DualInputPlanNode target = (DualInputPlanNode) c.getTarget(); JobVertex accessingVertex = this.vertices.get(target); TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration()); int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1; if (conf.getDriver().equals(JoinDriver.class)) { conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class); else if (conf.getDriver().equals(CoGroupDriver.class)) { conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class); new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration()); int inputIndex = headConfig.getDriverStrategy().getNumInputs(); headConfig.setIterationHeadSolutionSetInputIndex(inputIndex); translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false); new TaskConfig(container.getConfiguration()).addOutputShipStrategy(ShipStrategyType.FORWARD); chainedTask.getTaskConfig().setInputSerializer(inConn.getSerializer(), 0); final TaskConfig targetVertexConfig = new TaskConfig(targetVertex.getConfiguration()); targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0); } else if (node instanceof WorksetPlanNode) { WorksetPlanNode wspn = (WorksetPlanNode) node; targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0); targetVertexConfig.setIterationHeadSolutionSetInputIndex(1); } else { inConns = node.getInputs().iterator(); for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex); targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex); broadcastInputIndex += broadcastInputIndexDelta;
public void setHeadTask(JobVertex headTask, TaskConfig headConfig) { this.headTask = headTask; this.headFinalResultConfig = new TaskConfig(new Configuration()); // check if we already had a configuration, for example if the solution set was if (this.headConfig != null) { headConfig.getConfiguration().addAll(this.headConfig.getConfiguration()); } this.headConfig = headConfig; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration()); final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig(); final int numStepFunctionOuts = headConfig.getNumOutputs(); final int numFinalOuts = headFinalOutputConfig.getNumOutputs(); headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig); headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts); final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask(); if (relativeMemForBackChannel <= 0) { throw new CompilerException("Bug: No memory has been assigned to the iteration back channel."); headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel); final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration()); syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism()); syncConfig.setNumberOfIterations(maxNumIterations); tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration()); tailConfig.setIsWorksetUpdate(); tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel()); tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration()); tailConfigOfTerminationCriterion.setIsSolutionSetUpdate(); tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
final int outputIndex = sourceConfig.getNumOutputs(); sourceConfig.addOutputShipStrategy(channel.getShipStrategy()); if (outputIndex == 0) { sourceConfig.setOutputSerializer(channel.getSerializer()); sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex); sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex); } else { throw new RuntimeException("Range partitioning requires data distribution."); sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex); } else { throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set."); targetConfig.addBroadcastInputToGroup(inputNumber); } else { targetConfig.addInputToGroup(inputNumber);
this.config = new TaskConfig(taskConf); final Class<? extends Driver<S, OT>> driverClass = this.config.getDriver(); this.driver = InstantiationUtil.instantiate(driverClass, Driver.class); .addOperator(headName.startsWith("CHAIN") ? headName.substring(6) : headName); this.metrics.getIOMetricGroup().reuseInputMetricsForTask(); if (config.getNumberOfChainedStubs() == 0) { this.metrics.getIOMetricGroup().reuseOutputMetricsForTask(); int numInputs = driver.getNumberOfInputs(); int numComparators = driver.getNumberOfDriverComparators(); int numBroadcastInputs = this.config.getNumBroadcastInputs(); final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeGate(i); final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeBroadcastGate(i); for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) { final String name = this.config.getBroadcastInputName(i); readAndSetBroadcastInput(i, name, this.runtimeUdfContext, 1 /* superstep one for the start */);
throws Exception final int numOutputs = config.getNumOutputs(); final int numChained = config.getNumberOfChainedStubs(); if (numChained > 0) { if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) { throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection."); Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i); ct = ctc.newInstance(); final TaskConfig chainedStubConf = config.getChainedStubConfig(i); final String taskName = config.getChainedTaskName(i); previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
@Override public void prepare() throws Exception { final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy(); if(driverStrategy != DriverStrategy.ALL_GROUP_COMBINE){ throw new Exception("Invalid strategy " + driverStrategy + " for " + "GroupCombine."); } ExecutionConfig executionConfig = taskContext.getExecutionConfig(); this.objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (LOG.isDebugEnabled()) { LOG.debug("GroupCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
TaskConfig t = new TaskConfig(tic.getContainingVertex().getConfiguration()); t.addChainedTask(tic.getChainedTask(), tic.getTaskConfig(), tic.getTaskName());
chainedOperatorVertexIds, userDefinedChainedOperatorVertexIds); TaskConfig taskConfig = new TaskConfig(jobVertex.getConfiguration()); taskConfig.setStubWrapper(new UserCodeObjectWrapper<Object>(streamNode.getInputFormat())); } else { jobVertex = new JobVertex(
@Override protected void initOutputs() throws Exception { // initialize the regular outputs first (the ones into the step function). super.initOutputs(); // at this time, the outputs to the step function are created // add the outputs for the final solution List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>(); final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig(); final ClassLoader userCodeClassLoader = getUserCodeClassLoader(); this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig, userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs()); // sanity check the setup final int writersIntoStepFunction = this.eventualOutputs.size(); final int writersIntoFinalResult = finalOutputWriters.size(); final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput(); if (writersIntoStepFunction + writersIntoFinalResult != syncGateIndex) { throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates."); } // now, we can instantiate the sync gate this.toSync = getEnvironment().getWriter(syncGateIndex); }
targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex); } else { targetVertexConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(inputIndex); } else { sourceVertexConfig = new TaskConfig(sourceVertex.getConfiguration()); targetVertexConfig.setBroadcastGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal); } else { targetVertexConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(inputIndex, numDynamicSenderTasksTotal);
final boolean async = this.config.isInputAsynchronouslyMaterialized(i); final boolean cached = this.config.isInputCached(i); memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i)); if (memoryPages <= 0) { throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
headConfig = new TaskConfig(headVertex.getConfiguration()); toReturn = null; } else { headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration()); headConfig.setDriver(NoOpDriver.class); toReturn = headVertex; headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
@Test public void testTempInIterationTest() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0); DataSet<Tuple2<Long, Long>> update = iteration.getWorkset() .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); iteration.closeWith(update, update) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jg = jgg.compileJobGraph(oPlan); boolean solutionSetUpdateChecked = false; for(JobVertex v : jg.getVertices()) { if(v.getName().equals("SolutionSet Delta")) { // check if input of solution set delta is temped TaskConfig tc = new TaskConfig(v.getConfiguration()); assertTrue(tc.isInputAsynchronouslyMaterialized(0)); solutionSetUpdateChecked = true; } } assertTrue(solutionSetUpdateChecked); }
headConfig = new TaskConfig(headVertex.getConfiguration()); toReturn = null; } else { headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration()); headConfig.setDriver(NoOpDriver.class); toReturn = headVertex;