public static String getLocalStrategyString(LocalStrategy localStrategy) { if (localStrategy == null) { return "(null)"; } switch (localStrategy) { case NONE: return "(none)"; case SORT: return "Sort"; case COMBININGSORT: return "Sort (combining)"; default: return localStrategy.name(); } }
private void assignDriverResources(PlanNode node, TaskConfig config) { final double relativeMem = node.getRelativeMemoryPerSubTask(); if (relativeMem > 0) { config.setRelativeMemoryDriver(relativeMem); config.setFilehandlesDriver(this.defaultMaxFan); config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold); } }
private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) { if (c.getRelativeMemoryLocalStrategy() > 0) { config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy()); config.setFilehandlesInput(inputNum, this.defaultMaxFan); config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold); config.setUseLargeRecordHandler(this.useLargeRecordHandler); } }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) { config.setBroadcastInputSerializer(channel.getSerializer(), inputNum); config.setInputSerializer(channel.getSerializer(), inputNum); config.setInputLocalStrategy(inputNum, channel.getLocalStrategy()); if (channel.getLocalStrategyComparator() != null) { config.setInputComparator(channel.getLocalStrategyComparator(), inputNum); config.setInputAsynchronouslyMaterialized(inputNum, true); needsMemory = true; config.setInputCached(inputNum, true); needsMemory = true; throw new CompilerException("Bug in compiler: Inconsistent description of input materialization."); config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
headConfig = new TaskConfig(headVertex.getConfiguration()); toReturn = null; } else { headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration()); headConfig.setDriver(NoOpDriver.class); toReturn = headVertex; headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
public void setHeadTask(JobVertex headTask, TaskConfig headConfig) { this.headTask = headTask; this.headFinalResultConfig = new TaskConfig(new Configuration()); // check if we already had a configuration, for example if the solution set was if (this.headConfig != null) { headConfig.getConfiguration().addAll(this.headConfig.getConfiguration()); } this.headConfig = headConfig; }
@Test public void testTempInIterationTest() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0); DataSet<Tuple2<Long, Long>> update = iteration.getWorkset() .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); iteration.closeWith(update, update) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jg = jgg.compileJobGraph(oPlan); boolean solutionSetUpdateChecked = false; for(JobVertex v : jg.getVertices()) { if(v.getName().equals("SolutionSet Delta")) { // check if input of solution set delta is temped TaskConfig tc = new TaskConfig(v.getConfiguration()); assertTrue(tc.isInputAsynchronouslyMaterialized(0)); solutionSetUpdateChecked = true; } } assertTrue(solutionSetUpdateChecked); }
headConfig = new TaskConfig(headVertex.getConfiguration()); toReturn = null; } else { headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration()); headConfig.setDriver(NoOpDriver.class); toReturn = headVertex;
if (c.getLocalStrategy().dams()) { final double relativeMem = 1.0 / this.memoryConsumerWeights; c.setRelativeMemoryLocalStrategy(relativeMem);
public TimeCharacteristic getTimeCharacteristic() { int ordinal = config.getInteger(TIME_CHARACTERISTIC, -1); if (ordinal >= 0) { return TimeCharacteristic.values()[ordinal]; } else { throw new CorruptConfigurationException("time characteristic is not set"); } }
if (channel.getLocalStrategy().requiresComparator()) { channel.setLocalStrategyComparator(createComparator(type, channel.getLocalStrategyKeys(), getSortOrders(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder())));
config = new TaskConfig(new Configuration()); this.chainedTasks.put(node, new TaskInChain(node, ds.getPushChainDriverClass(), config, taskName)); } else { vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); config = new TaskConfig(vertex.getConfiguration()); config.setDriver(ds.getDriverClass()); config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setDriverStrategy(ds); for (int i = 0; i < ds.getNumRequiredComparators(); i++) { config.setDriverComparator(node.getComparator(i), i);
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { // NOT_FOUND // check the broadcast inputs for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) { // broadcast inputs are always dams return FOUND_SOURCE_AND_DAM; } } return NOT_FOUND; } }
if (this.input1.getLocalStrategy().dams() || this.input1.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; if (this.input2.getLocalStrategy().dams() || this.input2.getTempMode().breaksPipeline() || getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM;
boolean damOnAllRightPaths = true; if (strategy.firstDam() == DamBehavior.FULL_DAM || in1.getLocalStrategy().dams() || in1.getTempMode().breaksPipeline()) { someDamOnLeftPaths = true; } else { if (strategy.secondDam() == DamBehavior.FULL_DAM || in2.getLocalStrategy().dams() || in2.getTempMode().breaksPipeline()) { someDamOnRightPaths = true; } else {