public StreamNode getSourceVertex(StreamEdge edge) { return streamNodes.get(edge.getSourceId()); }
public List<Integer> getInEdgeIndices() { List<Integer> inEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : inEdges) { inEdgeIndices.add(edge.getSourceId()); } return inEdgeIndices; }
public void addOutEdge(StreamEdge outEdge) { if (outEdge.getSourceId() != getId()) { throw new IllegalArgumentException("Source id doesn't match the StreamNode id"); } else { outEdges.add(outEdge); } }
private void visitIteration(ArrayNode jsonArray, List<Integer> toVisit, int headId, Map<Integer, Integer> edgeRemapings, ArrayNode iterationInEdges) { Integer vertexID = toVisit.get(0); StreamNode vertex = streamGraph.getStreamNode(vertexID); toVisit.remove(vertexID); // Ignoring head and tail to avoid redundancy if (!streamGraph.vertexIDtoLoopTimeout.containsKey(vertexID)) { ObjectNode obj = mapper.createObjectNode(); jsonArray.add(obj); decorateNode(vertexID, obj); ArrayNode inEdges = mapper.createArrayNode(); obj.put(PREDECESSORS, inEdges); for (StreamEdge inEdge : vertex.getInEdges()) { int inputID = inEdge.getSourceId(); if (edgeRemapings.keySet().contains(inputID)) { decorateEdge(inEdges, inEdge, inputID); } else if (!streamGraph.vertexIDtoLoopTimeout.containsKey(inputID)) { decorateEdge(iterationInEdges, inEdge, inputID); } } edgeRemapings.put(vertexID, headId); visitIteration(jsonArray, toVisit, headId, edgeRemapings, iterationInEdges); } }
@VisibleForTesting public static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters( StreamConfig configuration, Environment environment) { List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); recordWriters.add( createRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return recordWriters; }
if (!hashes.containsKey(inEdge.getSourceId())) { return false;
recordWriters.get(i), outEdge, chainedConfigs.get(outEdge.getSourceId()), containingTask.getEnvironment());
getStreamNode(edge.getSourceId()).addOutEdge(edge); getStreamNode(edge.getTargetId()).addInEdge(edge);
byte[] otherHash = hashes.get(inEdge.getSourceId());
int inputID = inEdge.getSourceId(); Integer iterationHead = -1; for (StreamEdge inEdge : vertex.getInEdges()) { int operator = inEdge.getSourceId();
if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapShuffle")) { assertTrue(edge.getPartitioner() instanceof ShufflePartitioner); if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("ParallelizeMapForward")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); if (graph.getStreamNode(edge.getSourceId()).getOperatorName().equals("EvenOddSourceMap")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getSelectedNames().contains("even"));
if (edge.getSourceId() == input2.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else if (edge.getSourceId() == input3.getId()) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else { if (edge.getSourceId() == input4.getId()) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner); } else if (edge.getSourceId() == input5.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else {
public List<Integer> getInEdgeIndices() { List<Integer> inEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : inEdges) { inEdgeIndices.add(edge.getSourceId()); } return inEdgeIndices; }
public List<Integer> getInEdgeIndices() { List<Integer> inEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : inEdges) { inEdgeIndices.add(edge.getSourceId()); } return inEdgeIndices; }
public List<Integer> getInEdgeIndices() { List<Integer> inEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : inEdges) { inEdgeIndices.add(edge.getSourceId()); } return inEdgeIndices; }
public void addOutEdge(StreamEdge outEdge) { if (outEdge.getSourceId() != getId()) { throw new IllegalArgumentException("Source id doesn't match the StreamNode id"); } else { outEdges.add(outEdge); } }
public void addOutEdge(StreamEdge outEdge) { if (outEdge.getSourceId() != getId()) { throw new IllegalArgumentException("Source id doesn't match the StreamNode id"); } else { outEdges.add(outEdge); } }
public void addOutEdge(StreamEdge outEdge) { if (outEdge.getSourceId() != getId()) { throw new IllegalArgumentException("Source id doesn't match the StreamNode id"); } else { outEdges.add(outEdge); } }
@VisibleForTesting public static <OUT> List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createStreamRecordWriters( StreamConfig configuration, Environment environment) { List<StreamRecordWriter<SerializationDelegate<StreamRecord<OUT>>>> streamRecordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); streamRecordWriters.add( createStreamRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return streamRecordWriters; }
public void apply(StreamEdge edge, StreamGraph graph) { if (shipStrategy.equalsIgnoreCase(FORWARD_STRATEGY)) { if (graph.getStreamNode(edge.getTargetId()).getParallelism() != graph.getStreamNode(edge.getSourceId()).getParallelism()) { edge.setPartitioner(new RebalancePartitioner<>()); } else { edge.setPartitioner(new ForwardPartitioner<>()); } } else if (shipStrategy.equalsIgnoreCase(RESCALE_STRATEGY)) { edge.setPartitioner(new RescalePartitioner<>()); } else if (shipStrategy.equalsIgnoreCase(REBALANCE_STRATEGY)) { edge.setPartitioner(new RebalancePartitioner<>()); } }