if (partitioner == null && upstreamNode.getParallelism() == downstreamNode.getParallelism()) { partitioner = new ForwardPartitioner<Object>(); } else if (partitioner == null) { if (upstreamNode.getParallelism() != downstreamNode.getParallelism()) { throw new UnsupportedOperationException("Forward partitioning does not allow " + "change of parallelism. Upstream operation: " + upstreamNode + " parallelism: " + upstreamNode.getParallelism() + ", downstream operation: " + downstreamNode + " parallelism: " + downstreamNode.getParallelism() + " You must use another partitioning strategy, such as broadcast, rebalance, shuffle or global.");
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
int parallelism = streamNode.getParallelism();
"parallelism: " + node.getParallelism() + ", " + "user function: " + udfClassName + "}");
obj.put(ID, iterationHead); obj.put(PACT, "IterativeDataStream"); obj.put(PARALLELISM, streamGraph.getStreamNode(iterationHead).getParallelism()); obj.put(CONTENTS, "Stream Iteration"); ArrayNode iterationInputs = mapper.createArrayNode();
private void decorateNode(Integer vertexID, ObjectNode node) { StreamNode vertex = streamGraph.getStreamNode(vertexID); node.put(ID, vertexID); node.put(TYPE, vertex.getOperatorName()); if (streamGraph.getSourceIDs().contains(vertexID)) { node.put(PACT, "Data Source"); } else if (streamGraph.getSinkIDs().contains(vertexID)) { node.put(PACT, "Data Sink"); } else { node.put(PACT, "Operator"); } StreamOperator<?> operator = streamGraph.getStreamNode(vertexID).getOperator(); node.put(CONTENTS, vertex.getOperatorName()); node.put(PARALLELISM, streamGraph.getStreamNode(vertexID).getParallelism()); }
assertEquals(1, env.getStreamGraph().getStreamNode(src.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(map.getId()).getParallelism()); assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism()); assertEquals(1, env.getStreamGraph().getStreamNode(src.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(map.getId()).getParallelism()); assertEquals(1, env.getStreamGraph().getStreamNode(windowed.getId()).getParallelism()); assertEquals(10, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism()); assertEquals(7, env.getStreamGraph().getStreamNode(parallelSource.getId()).getParallelism()); assertEquals(3, env.getStreamGraph().getStreamNode(parallelSource.getId()).getParallelism()); assertEquals(2, env.getStreamGraph().getStreamNode(map.getId()).getParallelism()); assertEquals(4, env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getParallelism());
@Test @SuppressWarnings("unchecked") public void testFromCollectionParallelism() { try { TypeInformation<Integer> typeInfo = BasicTypeInfo.INT_TYPE_INFO; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Integer> dataStream1 = env.fromCollection(new DummySplittableIterator<Integer>(), typeInfo); try { dataStream1.setParallelism(4); fail("should throw an exception"); } catch (IllegalArgumentException e) { // expected } dataStream1.addSink(new DiscardingSink<Integer>()); DataStreamSource<Integer> dataStream2 = env.fromParallelCollection(new DummySplittableIterator<Integer>(), typeInfo).setParallelism(4); dataStream2.addSink(new DiscardingSink<Integer>()); env.getExecutionPlan(); assertEquals("Parallelism of collection source must be 1.", 1, env.getStreamGraph().getStreamNode(dataStream1.getId()).getParallelism()); assertEquals("Parallelism of parallel collection source must be 4.", 4, env.getStreamGraph().getStreamNode(dataStream2.getId()).getParallelism()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
assertEquals(3, itSink.getInEdges().size()); assertEquals(itSource.getParallelism(), itSink.getParallelism()); assertEquals(4, graph.getTargetVertex(edge).getParallelism()); } else if (graph.getTargetVertex(edge).getOperatorName().equals("shuffle")) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); assertEquals(2, graph.getTargetVertex(edge).getParallelism());
assertEquals(3, itSink.getInEdges().size()); assertEquals(itSource.getParallelism(), itSink.getParallelism());
private void generateNodeLocalHash(StreamNode node, Hasher hasher, int id) { hasher.putInt(id); hasher.putInt(node.getParallelism()); if (node.getOperator() instanceof AbstractUdfStreamOperator) { String udfClassName = ((AbstractUdfStreamOperator<?, ?>) node.getOperator()) .getUserFunction().getClass().getName(); hasher.putString(udfClassName, Charset.forName("UTF-8")); } } }
public void apply(StreamEdge edge, StreamGraph graph) { if (shipStrategy.equalsIgnoreCase(FORWARD_STRATEGY)) { if (graph.getStreamNode(edge.getTargetId()).getParallelism() != graph.getStreamNode(edge.getSourceId()).getParallelism()) { edge.setPartitioner(new RebalancePartitioner<>()); } else { edge.setPartitioner(new ForwardPartitioner<>()); } } else if (shipStrategy.equalsIgnoreCase(RESCALE_STRATEGY)) { edge.setPartitioner(new RescalePartitioner<>()); } else if (shipStrategy.equalsIgnoreCase(REBALANCE_STRATEGY)) { edge.setPartitioner(new RebalancePartitioner<>()); } }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled) { StreamNode upStreamVertex = edge.getSourceVertex(); StreamNode downStreamVertex = edge.getTargetVertex(); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled) { StreamNode upStreamVertex = edge.getSourceVertex(); StreamNode downStreamVertex = edge.getTargetVertex(); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = edge.getSourceVertex(); StreamNode downStreamVertex = edge.getTargetVertex(); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = edge.getSourceVertex(); StreamNode downStreamVertex = edge.getTargetVertex(); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
private void decorateNode(Integer vertexID, ObjectNode node) { StreamNode vertex = streamGraph.getStreamNode(vertexID); node.put(ID, vertexID); node.put(TYPE, vertex.getOperatorName()); if (streamGraph.getSourceIDs().contains(vertexID)) { node.put(PACT, "Data Source"); } else if (streamGraph.getSinkIDs().contains(vertexID)) { node.put(PACT, "Data Sink"); } else { node.put(PACT, "Operator"); } StreamOperator<?> operator = streamGraph.getStreamNode(vertexID).getOperator(); node.put(CONTENTS, vertex.getOperatorName()); node.put(PARALLELISM, streamGraph.getStreamNode(vertexID).getParallelism()); }
private void decorateNode(Integer vertexID, JSONObject node) throws JSONException { StreamNode vertex = streamGraph.getStreamNode(vertexID); node.put(ID, vertexID); node.put(TYPE, vertex.getOperatorName()); if (streamGraph.getSourceIDs().contains(vertexID)) { node.put(PACT, "Data Source"); } else if (streamGraph.getSinkIDs().contains(vertexID)) { node.put(PACT, "Data Sink"); } else { node.put(PACT, "Operator"); } StreamOperator<?> operator = streamGraph.getStreamNode(vertexID).getOperator(); node.put(CONTENTS, vertex.getOperatorName()); node.put(PARALLELISM, streamGraph.getStreamNode(vertexID).getParallelism()); }
private void decorateNode(Integer vertexID, ObjectNode node) { StreamNode vertex = streamGraph.getStreamNode(vertexID); node.put(ID, vertexID); node.put(TYPE, vertex.getOperatorName()); if (streamGraph.getSourceIDs().contains(vertexID)) { node.put(PACT, "Data Source"); } else if (streamGraph.getSinkIDs().contains(vertexID)) { node.put(PACT, "Data Sink"); } else { node.put(PACT, "Operator"); } StreamOperator<?> operator = streamGraph.getStreamNode(vertexID).getOperator(); node.put(CONTENTS, vertex.getOperatorName()); node.put(PARALLELISM, streamGraph.getStreamNode(vertexID).getParallelism()); }