private static boolean isCustomPartitioned(List<StreamEdge> edges) { boolean result = true; for (StreamEdge edge: edges) { if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) { result = false; } } return result; }
StreamEdge edge = new StreamEdge(upstreamNode, downstreamNode, typeNumber, outputNames, partitioner, outputTag); getStreamNode(edge.getSourceId()).addOutEdge(edge); getStreamNode(edge.getTargetId()).addInEdge(edge);
final StreamEdge edge = outputPair.f1; List<String> selectedNames = edge.getSelectedNames();
builder.append("\nPartitioning:"); for (StreamEdge output : getNonChainedOutputs(cl)) { int outputname = output.getTargetId(); builder.append("\n\t").append(outputname).append(": ").append(output.getPartitioner());
int outputId = outputEdge.getTargetId(); StreamConfig chainedOpConfig = chainedConfigs.get(outputId); streamOutputs, allOperators, outputEdge.getOutputTag()); allOutputs.add(new Tuple2<>(output, outputEdge));
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled) { StreamNode upStreamVertex = edge.getSourceVertex(); StreamNode downStreamVertex = edge.getTargetVertex(); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
public StreamNode getSourceVertex(StreamEdge edge) { return streamNodes.get(edge.getSourceId()); }
assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else if (graph.getTargetVertex(edge).getOperatorName().equals("IterForwardMap")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) { assertTrue(edge.getPartitioner() instanceof ShufflePartitioner); if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getSelectedNames().contains("even"));
assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));
StreamNode chainedNode = outEdge.getTargetVertex(); byte[] otherHash = hashes.get(inEdge.getSourceId()); + inEdge.getSourceVertex() + ". Cannot generate hash for " + node + ".");
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); boolean hasBroadcast = false; for (StreamEdge edge: streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) { if (edge.getPartitioner() instanceof ForwardPartitioner) { hasForward = true; if (edge.getPartitioner() instanceof BroadcastPartitioner) { hasBroadcast = true; if (edge.getSourceId() == input2.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else if (edge.getSourceId() == input3.getId()) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else { fail("Wrong input edge."); if (edge.getSourceId() == input4.getId()) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner); } else if (edge.getSourceId() == input5.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else { fail("Wrong input edge.");
public List<Integer> getOutEdgeIndices() { List<Integer> outEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : outEdges) { outEdgeIndices.add(edge.getTargetId()); } return outEdgeIndices; }
byte[] otherHash = hashes.get(inEdge.getSourceId()); + inEdge.getSourceVertex() + ". Cannot generate hash for " + node + ".");
public void apply(StreamEdge edge, StreamGraph graph) { if (shipStrategy.equalsIgnoreCase(FORWARD_STRATEGY)) { if (graph.getStreamNode(edge.getTargetId()).getParallelism() != graph.getStreamNode(edge.getSourceId()).getParallelism()) { edge.setPartitioner(new RebalancePartitioner<>()); } else { edge.setPartitioner(new ForwardPartitioner<>()); } } else if (shipStrategy.equalsIgnoreCase(RESCALE_STRATEGY)) { edge.setPartitioner(new RescalePartitioner<>()); } else if (shipStrategy.equalsIgnoreCase(REBALANCE_STRATEGY)) { edge.setPartitioner(new RebalancePartitioner<>()); } }
private void removeEdge(StreamEdge edge) { edge.getSourceVertex().getOutEdges().remove(edge); edge.getTargetVertex().getInEdges().remove(edge); }
private <T> RecordWriterOutput<T> createStreamOutput( StreamEdge edge, StreamConfig upStreamConfig, int outputIndex, Environment taskEnvironment, String taskName) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } @SuppressWarnings("unchecked") StreamPartitioner<T> outputPartitioner = (StreamPartitioner<T>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task ", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = taskEnvironment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output = new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout()); output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup()); return new RecordWriterOutput<>(output, outSerializer, sideOutputTag, this); }
inputSerializer1); StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 1, inputSerializer2); StreamEdge streamEdge = new StreamEdge(sourceVertexDummy, targetVertexDummy, 2,
private RecordWriterOutput<OUT> createStreamOutput( RecordWriter<SerializationDelegate<StreamRecord<OUT>>> recordWriter, StreamEdge edge, StreamConfig upStreamConfig, Environment taskEnvironment) { OutputTag sideOutputTag = edge.getOutputTag(); // OutputTag, return null if not sideOutput TypeSerializer outSerializer = null; if (edge.getOutputTag() != null) { // side output outSerializer = upStreamConfig.getTypeSerializerSideOut( edge.getOutputTag(), taskEnvironment.getUserClassLoader()); } else { // main output outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader()); } return new RecordWriterOutput<>(recordWriter, outSerializer, sideOutputTag, this); }
int inputType = inEdges.get(i).getTypeNumber(); InputGate reader = getEnvironment().getInputGate(i); switch (inputType) {
StreamNode child = outEdge.getTargetVertex();