private static StreamOperator<?> getOperatorForDataStream(DataStream<?> dataStream) { StreamExecutionEnvironment env = dataStream.getExecutionEnvironment(); StreamGraph streamGraph = env.getStreamGraph(); return streamGraph.getStreamNode(dataStream.getId()).getOperator(); }
private static StreamOperator<?> getOperatorFromDataStream(DataStream<?> dataStream) { StreamExecutionEnvironment env = dataStream.getExecutionEnvironment(); StreamGraph streamGraph = env.getStreamGraph(); return streamGraph.getStreamNode(dataStream.getId()).getOperator(); }
assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner); assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId()); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even")); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1)); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd")); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2)); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even")); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
env.getStreamGraph().getStreamEdges(map.getId(), unionFilter.getId()); } catch (RuntimeException e) { fail(e.getMessage()); env.getStreamGraph().getStreamEdges(flatMap.getId(), unionFilter.getId()); } catch (RuntimeException e) { fail(e.getMessage()); List<OutputSelector<?>> outputSelectors = env.getStreamGraph().getStreamNode(unionFilter.getId()).getOutputSelectors(); assertEquals(1, outputSelectors.size()); assertEquals(outputSelector, outputSelectors.get(0)); DataStreamSink<Integer> sink = select.print(); StreamEdge splitEdge = env.getStreamGraph().getStreamEdges(unionFilter.getId(), sink.getTransformation().getId()).get(0); assertEquals("a", splitEdge.getSelectedNames().get(0)); StreamEdge newSplitEdge = env.getStreamGraph().getStreamEdges(unionFilter.getId(), sinkWithIdentifier.getTransformation().getId()).get(0); assertEquals("a", newSplitEdge.getSelectedNames().get(0)); env.getStreamGraph().getStreamEdges(map.getId(), coMap.getId()); } catch (RuntimeException e) { fail(e.getMessage()); env.getStreamGraph().getStreamEdges(flatMap.getId(), coMap.getId()); } catch (RuntimeException e) { fail(e.getMessage());
/** * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of * maximum parallelism. */ @Test public void testSetupOfKeyGroupPartitioner() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> source = env.fromElements(1, 2, 3); DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner(); }
assertTrue(streamGraph.getStreamNode(selfUnion.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(selfUnion.getId()).getInEdges()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges().size() == 2); boolean hasForward = false; boolean hasBroadcast = false; for (StreamEdge edge: streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) { if (edge.getPartitioner() instanceof ForwardPartitioner) { hasForward = true; assertTrue(streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) { if (edge.getSourceId() == input2.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else if (edge.getSourceId() == input3.getId()) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else { assertTrue(streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges().size() == 2); for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) { if (edge.getSourceId() == input4.getId()) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner); } else if (edge.getSourceId() == input5.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else {
/** * Tests that the max parallelism is properly set for connected * streams. */ @Test public void testMaxParallelismWithConnectedKeyedStream() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128); DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult = input1 .connect(input2) .keyBy(value -> value, value -> value) .map(new NoOpIntCoMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner(); StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner(); }
/** * Tests that the global and operator-wide max parallelism setting is respected. */ @Test public void testMaxParallelismForwarding() { int globalMaxParallelism = 42; int keyedResult2MaxParallelism = 17; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setMaxParallelism(globalMaxParallelism); DataStream<Integer> source = env.fromElements(1, 2, 3); DataStream<Integer> keyedResult1 = source.keyBy(value -> value).map(new NoOpIntMap()); DataStream<Integer> keyedResult2 = keyedResult1 .keyBy(value -> value) .map(new NoOpIntMap()) .setMaxParallelism(keyedResult2MaxParallelism); keyedResult2.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResult1Node = graph.getStreamNode(keyedResult1.getId()); StreamNode keyedResult2Node = graph.getStreamNode(keyedResult2.getId()); assertEquals(globalMaxParallelism, keyedResult1Node.getMaxParallelism()); assertEquals(keyedResult2MaxParallelism, keyedResult2Node.getMaxParallelism()); }
sinkMethod.invoke(sink, minResource7, preferredResource7); assertEquals(minResource1, env.getStreamGraph().getStreamNode(source1.getId()).getMinResources()); assertEquals(preferredResource1, env.getStreamGraph().getStreamNode(source1.getId()).getPreferredResources()); assertEquals(minResource2, env.getStreamGraph().getStreamNode(map1.getId()).getMinResources()); assertEquals(preferredResource2, env.getStreamGraph().getStreamNode(map1.getId()).getPreferredResources()); assertEquals(minResource3, env.getStreamGraph().getStreamNode(source2.getId()).getMinResources()); assertEquals(preferredResource3, env.getStreamGraph().getStreamNode(source2.getId()).getPreferredResources()); assertEquals(minResource4, env.getStreamGraph().getStreamNode(map2.getId()).getMinResources()); assertEquals(preferredResource4, env.getStreamGraph().getStreamNode(map2.getId()).getPreferredResources()); assertEquals(minResource5, env.getStreamGraph().getStreamNode(connected.getId()).getMinResources()); assertEquals(preferredResource5, env.getStreamGraph().getStreamNode(connected.getId()).getPreferredResources()); assertEquals(minResource6, env.getStreamGraph().getStreamNode(windowed.getId()).getMinResources()); assertEquals(preferredResource6, env.getStreamGraph().getStreamNode(windowed.getId()).getPreferredResources());
assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), id1))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), id2))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), id3))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), id4))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), pid1))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), pid2))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), pid3))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), pid4))); assertTrue(isCustomPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), cid1))); assertTrue(isCustomPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), cid2))); assertTrue(isCustomPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), cid3))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), downStreamId1))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src2.getId(), downStreamId1))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), downStreamId2))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src2.getId(), downStreamId2))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), downStreamId3))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src2.getId(), downStreamId3))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), downStreamId4))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src2.getId(), downStreamId4))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src1.getId(), downStreamId5))); assertTrue(isPartitioned(env.getStreamGraph().getStreamEdges(src2.getId(), downStreamId5)));
/** * Tests that the max parallelism is automatically set to the parallelism if it has not been * specified. */ @Test public void testAutoMaxParallelism() { int globalParallelism = 42; int mapParallelism = 17; int maxParallelism = 21; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(globalParallelism); DataStream<Integer> source = env.fromElements(1, 2, 3); DataStream<Integer> keyedResult1 = source.keyBy(value -> value).map(new NoOpIntMap()); DataStream<Integer> keyedResult2 = keyedResult1.keyBy(value -> value).map(new NoOpIntMap()).setParallelism(mapParallelism); DataStream<Integer> keyedResult3 = keyedResult2.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult4 = keyedResult3.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism).setParallelism(mapParallelism); keyedResult4.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResult3Node = graph.getStreamNode(keyedResult3.getId()); StreamNode keyedResult4Node = graph.getStreamNode(keyedResult4.getId()); assertEquals(maxParallelism, keyedResult3Node.getMaxParallelism()); assertEquals(maxParallelism, keyedResult4Node.getMaxParallelism()); }
assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism()); assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism());