edge.setShipStrategyName(shipStrategy); edge.setPreProcessingOperationName(localStrategy); edge.setOperatorLevelCachingDescription(caching);
assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());
for (int inputNum = 0; inputNum < inputs.size(); inputNum++) { JobEdge edge = inputs.get(inputNum); if (edge.getSource() == null) { continue; JobVertex predecessor = edge.getSource().getProducer(); String shipStrategy = edge.getShipStrategyName(); String preProcessingOperation = edge.getPreProcessingOperationName(); String operatorLevelCaching = edge.getOperatorLevelCachingDescription(); gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s, %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId(), edge.getDistributionPattern())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s), %s, %s", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName(), edge.getDistributionPattern(), edge.getSource().getResultType())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId()); Collection<ExecutionVertexID> consumerExecutionVertices = edge.getConsumerExecutionVertices(i);
private void addNodesThatHaveNoNewPredecessors(JobVertex start, List<JobVertex> target, Set<JobVertex> remaining) { // forward traverse over all produced data sets and all their consumers for (IntermediateDataSet dataSet : start.getProducedDataSets()) { for (JobEdge edge : dataSet.getConsumers()) { // a vertex can be added, if it has no predecessors that are still in the 'remaining' set JobVertex v = edge.getTarget(); if (!remaining.contains(v)) { continue; } boolean hasNewPredecessors = false; for (JobEdge e : v.getInputs()) { // skip the edge through which we came if (e == edge) { continue; } IntermediateDataSet source = e.getSource(); if (remaining.contains(source.getProducer())) { hasNewPredecessors = true; break; } } if (!hasNewPredecessors) { target.add(v); remaining.remove(v); addNodesThatHaveNoNewPredecessors(v, target, remaining); } } } }
if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId());
private void connect(Integer headOfChain, StreamEdge edge) { physicalEdgesInOrder.add(edge); Integer downStreamvertexID = edge.getTargetId(); JobVertex headVertex = jobVertices.get(headOfChain); JobVertex downStreamVertex = jobVertices.get(downStreamvertexID); StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration()); downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1); StreamPartitioner<?> partitioner = edge.getPartitioner(); JobEdge jobEdge; if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED); } else { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED); } // set strategy name so that web interface can show it. jobEdge.setShipStrategyName(partitioner.toString()); if (LOG.isDebugEnabled()) { LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID); } }
public void connectIdInput(IntermediateDataSetID dataSetId, DistributionPattern distPattern) { JobEdge edge = new JobEdge(dataSetId, this, distPattern); this.inputs.add(edge); }
public boolean hasNoConnectedInputs() { for (JobEdge edge : inputs) { if (!edge.isIdReference()) { return false; } } return true; }
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) { final DistributionPattern pattern = edge.getDistributionPattern(); final IntermediateResultPartition[] sourcePartitions = source.getPartitions(); ExecutionEdge[] edges; switch (pattern) { case POINTWISE: edges = connectPointwise(sourcePartitions, inputNumber); break; case ALL_TO_ALL: edges = connectAllToAll(sourcePartitions, inputNumber); break; default: throw new RuntimeException("Unrecognized distribution pattern."); } this.inputEdges[inputNumber] = edges; // add the consumers to the source // for now (until the receiver initiated handshake is in place), we need to register the // edges as the execution graph for (ExecutionEdge ee : edges) { ee.getSource().addConsumer(ee, consumerNumber); } }
for (int i = 0; i < producerJobVertex.getParallelism(); i++) { for (JobEdge edge : jobGraph.getResult(resultID).getConsumers()) { for (ExecutionVertexID vertexID : edge.getConsumerExecutionVertices(i)) { if (!vertexInputsMap.containsKey(vertexID)) { vertexInputsMap.put(vertexID, new HashMap<>()); jobGraph.getResult(resultID).getResultType().isPipelined()); if (edge.getDistributionPattern() == DistributionPattern.ALL_TO_ALL) { vertexInputsMap.get(vertexID).put(resultID, new AllToAllVertexInput(resultID, consumableThreshold));
for (int inputNum = 0; inputNum < inputs.size(); inputNum++) { JobEdge edge = inputs.get(inputNum); if (edge.getSource() == null) { continue; JobVertex predecessor = edge.getSource().getProducer(); String shipStrategy = edge.getShipStrategyName(); String preProcessingOperation = edge.getPreProcessingOperationName(); String operatorLevelCaching = edge.getOperatorLevelCachingDescription(); gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
private void addNodesThatHaveNoNewPredecessors(JobVertex start, List<JobVertex> target, Set<JobVertex> remaining) { // forward traverse over all produced data sets and all their consumers for (IntermediateDataSet dataSet : start.getProducedDataSets()) { for (JobEdge edge : dataSet.getConsumers()) { // a vertex can be added, if it has no predecessors that are still in the 'remaining' set JobVertex v = edge.getTarget(); if (!remaining.contains(v)) { continue; } boolean hasNewPredecessors = false; for (JobEdge e : v.getInputs()) { // skip the edge through which we came if (e == edge) { continue; } IntermediateDataSet source = e.getSource(); if (remaining.contains(source.getProducer())) { hasNewPredecessors = true; break; } } if (!hasNewPredecessors) { target.add(v); remaining.remove(v); addNodesThatHaveNoNewPredecessors(v, target, remaining); } } } }
if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId());
private void connect(Integer headOfChain, StreamEdge edge) { physicalEdgesInOrder.add(edge); Integer downStreamvertexID = edge.getTargetId(); JobVertex headVertex = jobVertices.get(headOfChain); JobVertex downStreamVertex = jobVertices.get(downStreamvertexID); StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration()); downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1); StreamPartitioner<?> partitioner = edge.getPartitioner(); JobEdge jobEdge; if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED); } else { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED); } // set strategy name so that web interface can show it. jobEdge.setShipStrategyName(partitioner.toString()); if (LOG.isDebugEnabled()) { LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID); } }
public void connectIdInput(IntermediateDataSetID dataSetId, DistributionPattern distPattern) { JobEdge edge = new JobEdge(dataSetId, this, distPattern); this.inputs.add(edge); }
public boolean hasNoConnectedInputs() { for (JobEdge edge : inputs) { if (!edge.isIdReference()) { return false; } } return true; }
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) { final DistributionPattern pattern = edge.getDistributionPattern(); final IntermediateResultPartition[] sourcePartitions = source.getPartitions(); ExecutionEdge[] edges; switch (pattern) { case POINTWISE: edges = connectPointwise(sourcePartitions, inputNumber); break; case ALL_TO_ALL: edges = connectAllToAll(sourcePartitions, inputNumber); break; default: throw new RuntimeException("Unrecognized distribution pattern."); } this.inputEdges[inputNumber] = edges; // add the consumers to the source // for now (until the receiver initiated handshake is in place), we need to register the // edges as the execution graph for (ExecutionEdge ee : edges) { ee.getSource().addConsumer(ee, consumerNumber); } }
for (int inputNum = 0; inputNum < inputs.size(); inputNum++) { JobEdge edge = inputs.get(inputNum); if (edge.getSource() == null) { continue; JobVertex predecessor = edge.getSource().getProducer(); String shipStrategy = edge.getShipStrategyName(); String preProcessingOperation = edge.getPreProcessingOperationName(); String operatorLevelCaching = edge.getOperatorLevelCachingDescription(); gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
edge.setShipStrategyName(shipStrategy); edge.setPreProcessingOperationName(localStrategy); edge.setOperatorLevelCachingDescription(caching);