@Override public JobExecutionResult execute(String jobName) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile : jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return jobExecutor.executeJobBlocking(jobGraph); }
public void setParallelism(Integer vertexID, int parallelism) { if (getStreamNode(vertexID) != null) { getStreamNode(vertexID).setParallelism(parallelism); } }
private void configureCheckpointing() { CheckpointConfig cfg = streamGraph.getCheckpointConfig(); ExecutionConfig executionConfig = streamGraph.getExecutionConfig(); for (StreamNode node : streamGraph.getStreamNodes()) { StreamOperator<?> op = node.getOperator(); if (op instanceof AbstractUdfStreamOperator) { if (streamGraph.getStateBackend() == null) { serializedStateBackend = null; } else { try { serializedStateBackend = new SerializedValue<StateBackend>(streamGraph.getStateBackend());
/** * Private constructor. The generator should only be invoked using {@link #generate}. */ private StreamGraphGenerator(StreamExecutionEnvironment env) { this.streamGraph = new StreamGraph(env); this.streamGraph.setChaining(env.isChainingEnabled()); this.streamGraph.setStateBackend(env.getStateBackend()); this.env = env; this.alreadyTransformed = new HashMap<>(); }
public <T> void addOutputSelector(Integer vertexID, OutputSelector<T> outputSelector) { if (virtualPartitionNodes.containsKey(vertexID)) { addOutputSelector(virtualPartitionNodes.get(vertexID).f0, outputSelector); } else if (virtualSelectNodes.containsKey(vertexID)) { addOutputSelector(virtualSelectNodes.get(vertexID).f0, outputSelector); } else { getStreamNode(vertexID).addOutputSelector(outputSelector); if (LOG.isDebugEnabled()) { LOG.debug("Outputselector set for {}", vertexID); } } }
/** * Determines the slot sharing group of an operation across virtual nodes. */ public String getSlotSharingGroup(Integer id) { if (virtualSideOutputNodes.containsKey(id)) { Integer mappedId = virtualSideOutputNodes.get(id).f0; return getSlotSharingGroup(mappedId); } else if (virtualSelectNodes.containsKey(id)) { Integer mappedId = virtualSelectNodes.get(id).f0; return getSlotSharingGroup(mappedId); } else if (virtualPartitionNodes.containsKey(id)) { Integer mappedId = virtualPartitionNodes.get(id).f0; return getSlotSharingGroup(mappedId); } else { StreamNode node = getStreamNode(id); return node.getSlotSharingGroup(); } }
/** * Executes the remote job. * * @param streamGraph * Stream Graph to execute * @param jarFiles * List of jar file URLs to ship to the cluster * @return The result of the job execution, containing elapsed time and accumulators. */ @Override protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException { URL jarUrl; try { jarUrl = flinkILoop.writeFilesToDisk().getAbsoluteFile().toURI().toURL(); } catch (MalformedURLException e) { throw new ProgramInvocationException("Could not write the user code classes to disk.", streamGraph.getJobGraph().getJobID(), e); } List<URL> allJarFiles = new ArrayList<>(jarFiles.size() + 1); allJarFiles.addAll(jarFiles); allJarFiles.add(jarUrl); return super.executeRemotely(streamGraph, allJarFiles); }
@Override public JobExecutionResult execute(String jobName) throws ProgramInvocationException { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); return executeRemotely(streamGraph, jarFiles); }
@Override public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); if (env instanceof OptimizerPlanEnvironment) { ((OptimizerPlanEnvironment) env).setPlan(streamGraph); } else if (env instanceof PreviewPlanEnvironment) { ((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON()); } throw new OptimizerPlanEnvironment.ProgramAbortException(); } }
@Override public void dumpStreamingPlanAsJSON(File file) throws IOException { PrintWriter pw = null; try { pw = new PrintWriter(new FileOutputStream(file), false); pw.write(getStreamingPlanAsJSON()); pw.flush(); } finally { if (pw != null) { pw.close(); } } } }
private void decorateNode(Integer vertexID, ObjectNode node) { StreamNode vertex = streamGraph.getStreamNode(vertexID); node.put(ID, vertexID); node.put(TYPE, vertex.getOperatorName()); if (streamGraph.getSourceIDs().contains(vertexID)) { node.put(PACT, "Data Source"); } else if (streamGraph.getSinkIDs().contains(vertexID)) { node.put(PACT, "Data Sink"); } else { node.put(PACT, "Operator"); } StreamOperator<?> operator = streamGraph.getStreamNode(vertexID).getOperator(); node.put(CONTENTS, vertex.getOperatorName()); node.put(PARALLELISM, streamGraph.getStreamNode(vertexID).getParallelism()); }
@Override public Map<Integer, byte[]> traverseStreamGraphAndGenerateHashes(StreamGraph streamGraph) { HashMap<Integer, byte[]> hashResult = new HashMap<>(); for (StreamNode streamNode : streamGraph.getStreamNodes()) { String userHash = streamNode.getUserHash(); if (null != userHash) { hashResult.put(streamNode.getId(), StringUtils.hexStringToByte(userHash)); } } return hashResult; } }
for (StreamNode node : graph.getStreamNodes()) { edgesInStreamGraph += node.getInEdges().size(); transformationIdMap.put(node.getTransformationUID(), node.getId()); graph.getStreamNodes().size() != property.getStreamNodeProperties().size()) { throw new RuntimeException("Node or edge number in stream graph and json file not matched, please generate a new json."); throw new RuntimeException("Fail to apply resource configuration file, node " + nodeProperty.toString() + " not found."); StreamNode node = graph.getStreamNode(nodeId); if (node == null) { throw new RuntimeException("Fail to apply resource configuration file, node " + nodeProperty.toString() + " not found."); StreamEdge edge = graph.getStreamEdges( transformationIdMap.get(edgeProperty.getSource()), transformationIdMap.get(edgeProperty.getTarget())).get(edgeProperty.getIndex());
/** * Set uid for streamNode in the streamGraph. * @param streamGraph SteamGraph. */ public static Map<String, StreamNode> setUid(StreamGraph streamGraph) { Map<String, StreamNode> existedUidMap = new HashMap<>(); AtomicInteger atomicInteger = new AtomicInteger(0); Set<Integer> orderNodeIdSet = new TreeSet<>(); for (StreamNode streamNode : streamGraph.getStreamNodes()) { orderNodeIdSet.add(streamNode.getId()); } for (StreamNode streamNode : streamGraph.getStreamNodes()) { if (streamNode.getTransformationUID() != null) { existedUidMap.put(streamNode.getTransformationUID(), streamNode); } } for (Integer nodeId : orderNodeIdSet) { StreamNode node = streamGraph.getStreamNode(nodeId); String uid = "table-" + atomicInteger.getAndIncrement(); while (existedUidMap.containsKey(uid)) { uid = "table-" + atomicInteger.getAndIncrement(); } setNodeUID(node, uid); } return existedUidMap; }
/** * Tests that disabled checkpointing sets the checkpointing interval to Long.MAX_VALUE. */ @Test public void testDisabledCheckpointing() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamGraph streamGraph = new StreamGraph(env); assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled()); JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph); JobCheckpointingSettings snapshottingSettings = jobGraph.getCheckpointingSettings(); assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointCoordinatorConfiguration().getCheckpointInterval()); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); // define the dataflow StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(2); env.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, 1000)); env.readFileStream("input/", 60000, FileMonitoringFunction.WatchType.ONLY_NEW_FILES) .addSink(new DiscardingSink<String>()); // generate a job graph final JobGraph jobGraph = env.getStreamGraph().getJobGraph(); File jobGraphFile = new File(params.get("output", "job.graph")); try (FileOutputStream output = new FileOutputStream(jobGraphFile); ObjectOutputStream obOutput = new ObjectOutputStream(output)){ obOutput.writeObject(jobGraph); } } }
@Override public JobExecutionResult execute(String jobName) throws Exception { Preconditions.checkNotNull(jobName, "Streaming Job name should not be null."); StreamGraph streamGraph = this.getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); // execute the programs if (ctx instanceof DetachedEnvironment) { LOG.warn("Job was executed in detached mode, the results will be available on completion."); ((DetachedEnvironment) ctx).setDetachedPlan(streamGraph); return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE; } else { return ctx .getClient() .run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()) .getJobExecutionResult(); } } }
@Override public JobExecutionResult execute(String jobName) throws Exception { StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); transformations.clear(); if (env instanceof OptimizerPlanEnvironment) { ((OptimizerPlanEnvironment) env).setPlan(streamGraph); } else if (env instanceof PreviewPlanEnvironment) { ((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON()); } throw new OptimizerPlanEnvironment.ProgramAbortException(); } }
/** * Private constructor. The generator should only be invoked using {@link #generate}. */ private StreamGraphGenerator(StreamExecutionEnvironment env) { this.streamGraph = new StreamGraph(env); this.streamGraph.setChaining(env.isChainingEnabled()); this.streamGraph.setStateBackend(env.getStateBackend()); this.env = env; this.alreadyTransformed = new HashMap<>(); }
/** * Creates the plan with which the system will execute the program, and * returns it as a String using a JSON representation of the execution data * flow graph. Note that this needs to be called, before the plan is * executed. * * @return The execution plan of the program, as a JSON String. */ public String getExecutionPlan() { return getStreamGraph().getStreamingPlanAsJSON(); }