@Test public void testInheritOverride() { // verify that we can explicitly disable inheritance of the input slot sharing groups StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should not inherit group but be in "default" src1.union(src2).filter(dummyFilter).slotSharingGroup("default"); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); }
Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2); JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
JobGraph jobGraph = jgg.compileJobGraph(op); JobVertex sourceMapVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0); JobVertex iterationHeadVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1); JobVertex deltaVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(2); JobVertex iterationTailVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(3); JobVertex feedbackVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(4); JobVertex sinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(5); JobVertex iterationSyncVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(6);
@Test public void testUnion() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1"); // this should not inherit group "src-1" src1.union(src2).filter(dummyFilter); DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should inherit "group-1" now src3.union(src4).filter(dummyFilter); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); // first pipeline assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); // second pipeline assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup()); assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); }
Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2); JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
/** * Tests that there are no collisions with two identical sources. * * <pre> * [ (src0) ] --\ * +--> [ (sink) ] * [ (src1) ] --/ * </pre> */ @Test public void testNodeHashIdenticalSources() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); DataStream<String> src0 = env.addSource(new NoOpSourceFunction()); DataStream<String> src1 = env.addSource(new NoOpSourceFunction()); src0.union(src1).addSink(new NoOpSinkFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertTrue(vertices.get(0).isInputVertex()); assertTrue(vertices.get(1).isInputVertex()); assertNotNull(vertices.get(0).getID()); assertNotNull(vertices.get(1).getID()); assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID()); }
Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 3); return jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph()); List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources(); JobVertex sourceVertex = verticesSorted.get(0); JobVertex mapPrintVertex = verticesSorted.get(1);
streamGraph.setJobName("test job"); JobGraph jobGraph = streamGraph.getJobGraph(); List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
JobGraph jobGraph = jgg.compileJobGraph(op); JobVertex sourceMapFilterVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0); JobVertex iterationHeadVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1); JobVertex feedbackVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(2); JobVertex sinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(3); JobVertex iterationSyncVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(4);
JobVertex sourceMapFilterVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0); JobVertex reduceSinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
JobVertexID sourceId = jobGraph.getVerticesSortedTopologicallyFromSources() .get(0).getID(); JobVertexID unchainedSourceId = jobGraph.getVerticesSortedTopologicallyFromSources() .get(0).getID();
JobVertex chainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1); assertTrue(chainedMap.getName().startsWith("map")); JobVertexID chainedMapId = chainedMap.getID(); JobVertex unchainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1); assertEquals("map", unchainedMap.getName()); JobVertexID unchainedMapId = unchainedMap.getID();
final Configuration cfg = new Configuration(); List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources();
JobGraph jobGraph = jgg.compileJobGraph(optimizedPlan); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
@Override public void onSchedulingStarted() { final List<ExecutionVertexID> verticesToSchedule = new ArrayList<>(); for (JobVertex vertex : jobGraph.getVerticesSortedTopologicallyFromSources()) { for (int i = 0; i < vertex.getParallelism(); i++) { verticesToSchedule.add(new ExecutionVertexID(vertex.getID(), i)); } } scheduler.scheduleExecutionVertices(verticesToSchedule); }
@Override public void onSchedulingStarted() { final List<ExecutionVertexID> verticesToSchedule = new ArrayList<>(); for (JobVertex vertex : jobGraph.getVerticesSortedTopologicallyFromSources()) { if (vertex.isInputVertex()) { for (int i = 0; i < vertex.getParallelism(); i++) { verticesToSchedule.add(new ExecutionVertexID(vertex.getID(), i)); } } } scheduleOneByOne(verticesToSchedule); }