userDefinedChainedOperatorVertexIds); TaskConfig taskConfig = new TaskConfig(jobVertex.getConfiguration()); taskConfig.setStubWrapper(new UserCodeObjectWrapper<Object>(streamNode.getInputFormat())); } else { jobVertex = new JobVertex(
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters());
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
public static void setStubFormat(TaskConfig config, InputFormat inputFormat) { config.setStubWrapper(new UserCodeObjectWrapper<>(inputFormat)); }
public static void setStubFormat(TaskConfig config, OutputFormat outputFormat) { config.setStubWrapper(new UserCodeObjectWrapper<>(outputFormat)); }
public static void setStubFormats( TaskConfig config, @Nullable Map<OperatorID, InputFormat> idInputFormatMap, @Nullable Map<OperatorID, OutputFormat> idOutputFormatMap) { config.setStubWrapper(new UserCodeObjectWrapper<>(new Tuple2<>(idInputFormatMap, idOutputFormatMap))); }
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException { final OutputFormatVertex vertex = new OutputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSinkTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }