/** * Returns a {@link JobVertexID} to vertex name mapping for the given graph. */ private Map<JobVertexID, String> rememberIds(JobGraph jobGraph) { final Map<JobVertexID, String> ids = new HashMap<>(); for (JobVertex vertex : jobGraph.getVertices()) { ids.put(vertex.getID(), vertex.getName()); } return ids; }
/** * Verifies that no {@link JobVertexID} of the {@link JobGraph} is contained in the given map. */ private void verifyIdsNotEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { assertFalse(ids.containsKey(vertex.getID())); } }
/** * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map * and mapped to the same vertex name. */ private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { String expectedName = ids.get(vertex.getID()); assertNotNull(expectedName); assertEquals(expectedName, vertex.getName()); } }
/** * Tests that there are no collisions with two identical sources. * * <pre> * [ (src0) ] --\ * +--> [ (sink) ] * [ (src1) ] --/ * </pre> */ @Test public void testNodeHashIdenticalSources() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); DataStream<String> src0 = env.addSource(new NoOpSourceFunction()); DataStream<String> src1 = env.addSource(new NoOpSourceFunction()); src0.union(src1).addSink(new NoOpSinkFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertTrue(vertices.get(0).isInputVertex()); assertTrue(vertices.get(1).isInputVertex()); assertNotNull(vertices.get(0).getID()); assertNotNull(vertices.get(1).getID()); assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID()); }
/** * Tests that a changed operator name does not affect the hash. */ @Test public void testChangedOperatorName() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID(); env = StreamExecutionEnvironment.createLocalEnvironment(); env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction()); jobGraph = env.getStreamGraph().getJobGraph(); JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID(); assertEquals(expected, actual); }
ExecutionJobVertex execSourceVertex = eg.getJobVertex(sourceVertex.getID()); ExecutionJobVertex execMapVertex = eg.getJobVertex(mapVertex.getID()); ExecutionJobVertex execSinkVertex = eg.getJobVertex(sinkVertex.getID()); assertEquals(1, mapTaskVertex.getInputEdges(0).length); ExecutionEdge inputEdge = mapTaskVertex.getInputEdges(0)[0]; assertEquals(sourceVertex.getID(), inputEdge.getSource().getProducer().getJobvertexId()); int inputPartition = inputEdge.getSource().getPartitionNumber(); if (!mapInputPartitionCounts.containsKey(inputPartition)) { ExecutionEdge inputEdge1 = sinkTaskVertex.getInputEdges(0)[0]; ExecutionEdge inputEdge2 = sinkTaskVertex.getInputEdges(0)[1]; assertEquals(mapVertex.getID(), inputEdge1.getSource().getProducer().getJobvertexId()); assertEquals(mapVertex.getID(), inputEdge2.getSource().getProducer().getJobvertexId());
/** * Tests that there are no collisions with two identical intermediate nodes connected to the * same predecessor. * * <pre> * /-> [ (map) ] -> [ (sink) ] * [ (src) ] -+ * \-> [ (map) ] -> [ (sink) ] * </pre> */ @Test public void testNodeHashIdenticalNodes() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); DataStream<String> src = env.addSource(new NoOpSourceFunction()); src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction()); src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); Set<JobVertexID> vertexIds = new HashSet<>(); for (JobVertex vertex : jobGraph.getVertices()) { assertTrue(vertexIds.add(vertex.getID())); } }
@Override public JobVertexID getJobVertexId() { return jobVertex.getID(); }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
@Override public void updateJobGraph(JobGraph jobGraph) { for (JobVertex vertex : jobGraph.getVertices()) { if (jobVertexID.equals(vertex.getID())) { updateVertex(vertex); } } } }