public List<Integer> getOutEdgeIndices() { List<Integer> outEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : outEdges) { outEdgeIndices.add(edge.getTargetId()); } return outEdgeIndices; }
public StreamNode getTargetVertex(StreamEdge edge) { return streamNodes.get(edge.getTargetId()); }
public void addInEdge(StreamEdge inEdge) { if (inEdge.getTargetId() != getId()) { throw new IllegalArgumentException("Destination id doesn't match the StreamNode id"); } else { inEdges.add(inEdge); } }
private String createChainedName(Integer vertexID, List<StreamEdge> chainedOutputs) { String operatorName = streamGraph.getStreamNode(vertexID).getOperatorName(); if (chainedOutputs.size() > 1) { List<String> outputChainedNames = new ArrayList<>(); for (StreamEdge chainable : chainedOutputs) { outputChainedNames.add(chainedNames.get(chainable.getTargetId())); } return operatorName + " -> (" + StringUtils.join(outputChainedNames, ", ") + ")"; } else if (chainedOutputs.size() == 1) { return operatorName + " -> " + chainedNames.get(chainedOutputs.get(0).getTargetId()); } else { return operatorName; } }
public List<StreamEdge> getStreamEdges(int sourceId, int targetId) { List<StreamEdge> result = new ArrayList<>(); for (StreamEdge edge : getStreamNode(sourceId).getOutEdges()) { if (edge.getTargetId() == targetId) { result.add(edge); } } if (result.isEmpty()) { throw new RuntimeException("No such edge in stream graph: " + sourceId + " -> " + targetId); } return result; }
private void setPhysicalEdges() { Map<Integer, List<StreamEdge>> physicalInEdgesInOrder = new HashMap<Integer, List<StreamEdge>>(); for (StreamEdge edge : physicalEdgesInOrder) { int target = edge.getTargetId(); List<StreamEdge> inEdges = physicalInEdgesInOrder.get(target); // create if not set if (inEdges == null) { inEdges = new ArrayList<>(); physicalInEdgesInOrder.put(target, inEdges); } inEdges.add(edge); } for (Map.Entry<Integer, List<StreamEdge>> inEdges : physicalInEdgesInOrder.entrySet()) { int vertex = inEdges.getKey(); List<StreamEdge> edgeList = inEdges.getValue(); vertexConfigs.get(vertex).setInPhysicalEdges(edgeList); } }
private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources(); for (StreamEdge chainable : chainedOutputs) { preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId())); } return preferredResources; }
private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources(); for (StreamEdge chainable : chainedOutputs) { minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId())); } return minResources; }
builder.append("\nPartitioning:"); for (StreamEdge output : getNonChainedOutputs(cl)) { int outputname = output.getTargetId(); builder.append("\n\t").append(outputname).append(": ").append(output.getPartitioner());
private void connect(Integer headOfChain, StreamEdge edge) { physicalEdgesInOrder.add(edge); Integer downStreamvertexID = edge.getTargetId(); JobVertex headVertex = jobVertices.get(headOfChain); JobVertex downStreamVertex = jobVertices.get(downStreamvertexID); StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration()); downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1); StreamPartitioner<?> partitioner = edge.getPartitioner(); JobEdge jobEdge; if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED); } else { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED); } // set strategy name so that web interface can show it. jobEdge.setShipStrategyName(partitioner.toString()); if (LOG.isDebugEnabled()) { LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID); } }
createChain(startNodeId, chainable.getTargetId(), hashes, legacyHashes, chainIndex + 1, chainedOperatorHashes)); createChain(nonChainable.getTargetId(), nonChainable.getTargetId(), hashes, legacyHashes, 0, chainedOperatorHashes);
int outputId = outputEdge.getTargetId(); StreamConfig chainedOpConfig = chainedConfigs.get(outputId);
getStreamNode(edge.getTargetId()).addInEdge(edge);
public List<Integer> getOutEdgeIndices() { List<Integer> outEdgeIndices = new ArrayList<Integer>(); for (StreamEdge edge : outEdges) { outEdgeIndices.add(edge.getTargetId()); } return outEdgeIndices; }
public void addInEdge(StreamEdge inEdge) { if (inEdge.getTargetId() != getId()) { throw new IllegalArgumentException("Destination id doesn't match the StreamNode id"); } else { inEdges.add(inEdge); } }
public void addInEdge(StreamEdge inEdge) { if (inEdge.getTargetId() != getId()) { throw new IllegalArgumentException("Destination id doesn't match the StreamNode id"); } else { inEdges.add(inEdge); } }
public List<StreamEdge> getStreamEdges(int sourceId, int targetId) { List<StreamEdge> result = new ArrayList<>(); for (StreamEdge edge : getStreamNode(sourceId).getOutEdges()) { if (edge.getTargetId() == targetId) { result.add(edge); } } if (result.isEmpty()) { throw new RuntimeException("No such edge in stream graph: " + sourceId + " -> " + targetId); } return result; }
private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources(); for (StreamEdge chainable : chainedOutputs) { minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId())); } return minResources; }
private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources(); for (StreamEdge chainable : chainedOutputs) { minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId())); } return minResources; }
private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources(); for (StreamEdge chainable : chainedOutputs) { preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId())); } return preferredResources; }