private void removeVertex(StreamNode toRemove) { Set<StreamEdge> edgesToRemove = new HashSet<>(); edgesToRemove.addAll(toRemove.getInEdges()); edgesToRemove.addAll(toRemove.getOutEdges()); for (StreamEdge edge : edgesToRemove) { removeEdge(edge); } streamNodes.remove(toRemove.getId()); }
private void removeEdge(StreamEdge edge) { getSourceVertex(edge).getOutEdges().remove(edge); getTargetVertex(edge).getInEdges().remove(edge); }
private void visitIteration(ArrayNode jsonArray, List<Integer> toVisit, int headId, Map<Integer, Integer> edgeRemapings, ArrayNode iterationInEdges) { Integer vertexID = toVisit.get(0); StreamNode vertex = streamGraph.getStreamNode(vertexID); toVisit.remove(vertexID); // Ignoring head and tail to avoid redundancy if (!streamGraph.vertexIDtoLoopTimeout.containsKey(vertexID)) { ObjectNode obj = mapper.createObjectNode(); jsonArray.add(obj); decorateNode(vertexID, obj); ArrayNode inEdges = mapper.createArrayNode(); obj.put(PREDECESSORS, inEdges); for (StreamEdge inEdge : vertex.getInEdges()) { int inputID = inEdge.getSourceId(); if (edgeRemapings.keySet().contains(inputID)) { decorateEdge(inEdges, inEdge, inputID); } else if (!streamGraph.vertexIDtoLoopTimeout.containsKey(inputID)) { decorateEdge(iterationInEdges, inEdge, inputID); } } edgeRemapings.put(vertexID, headId); visitIteration(jsonArray, toVisit, headId, edgeRemapings, iterationInEdges); } }
for (StreamEdge inEdge : node.getInEdges()) {
|| Collections.disjoint(vertex.getInEdges(), toVisit)) { node.put(PREDECESSORS, inputs); for (StreamEdge inEdge : vertex.getInEdges()) { int inputID = inEdge.getSourceId(); } else { Integer iterationHead = -1; for (StreamEdge inEdge : vertex.getInEdges()) { int operator = inEdge.getSourceId();
for (StreamEdge inEdge : node.getInEdges()) { byte[] otherHash = hashes.get(inEdge.getSourceId());
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
/** * Tests that the max parallelism is properly set for connected * streams. */ @Test public void testMaxParallelismWithConnectedKeyedStream() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128); DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult = input1 .connect(input2) .keyBy(value -> value, value -> value) .map(new NoOpIntCoMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner(); StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner(); }
@Test public void testImmutabilityWithCoiteration() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance IterativeStream<Integer> iter1 = source.iterate(); // Calling withFeedbackType should create a new iteration ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class); iter1.closeWith(iter1.map(noOpIntMap)).print(); iter2.closeWith(iter2.map(noOpCoMap)).print(); StreamGraph graph = env.getStreamGraph(); assertEquals(2, graph.getIterationSourceSinkPairs().size()); for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) { assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)), graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0))); } }
/** * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of * maximum parallelism. */ @Test public void testSetupOfKeyGroupPartitioner() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> source = env.fromElements(1, 2, 3); DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner(); }
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner); assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());
assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2); boolean hasForward = false; boolean hasBroadcast = false; for (StreamEdge edge: streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) { if (edge.getPartitioner() instanceof ForwardPartitioner) { hasForward = true; assertTrue(streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) { if (edge.getSourceId() == input2.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) { if (edge.getSourceId() == input4.getId()) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner);
assertEquals(3, itSink.getInEdges().size()); for (StreamEdge edge : itSink.getInEdges()) { if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) { assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
assertEquals(3, itSink.getInEdges().size()); for (StreamEdge edge : itSink.getInEdges()) { String tailName = graph.getSourceVertex(edge).getOperatorName(); if (tailName.equals("split")) {
assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof ForwardPartitioner); assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStateKeySerializer()); assertEquals(key1, env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner1()); assertTrue(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof KeyGroupStreamPartitioner); assertTrue(env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof KeyGroupStreamPartitioner);
private void removeVertex(StreamNode toRemove) { Set<StreamEdge> edgesToRemove = new HashSet<>(); edgesToRemove.addAll(toRemove.getInEdges()); edgesToRemove.addAll(toRemove.getOutEdges()); for (StreamEdge edge : edgesToRemove) { removeEdge(edge); } streamNodes.remove(toRemove.getId()); }
private void removeVertex(StreamNode toRemove) { Set<StreamEdge> edgesToRemove = new HashSet<>(); edgesToRemove.addAll(toRemove.getInEdges()); edgesToRemove.addAll(toRemove.getOutEdges()); for (StreamEdge edge : edgesToRemove) { removeEdge(edge); } streamNodes.remove(toRemove.getId()); }
private void removeVertex(StreamNode toRemove) { Set<StreamEdge> edgesToRemove = new HashSet<>(); edgesToRemove.addAll(toRemove.getInEdges()); edgesToRemove.addAll(toRemove.getOutEdges()); for (StreamEdge edge : edgesToRemove) { removeEdge(edge); } streamNodes.remove(toRemove.getId()); }
private void removeEdge(StreamEdge edge) { edge.getSourceVertex().getOutEdges().remove(edge); edge.getTargetVertex().getInEdges().remove(edge); }