assertEquals(1, mapTaskVertex.getInputEdges(0).length); ExecutionEdge inputEdge = mapTaskVertex.getInputEdges(0)[0]; assertEquals(sourceVertex.getID(), inputEdge.getSource().getProducer().getJobvertexId()); int inputPartition = inputEdge.getSource().getPartitionNumber(); if (!mapInputPartitionCounts.containsKey(inputPartition)) { mapInputPartitionCounts.put(inputPartition, 1); ExecutionEdge inputEdge1 = sinkTaskVertex.getInputEdges(0)[0]; ExecutionEdge inputEdge2 = sinkTaskVertex.getInputEdges(0)[1]; assertEquals(mapVertex.getID(), inputEdge1.getSource().getProducer().getJobvertexId()); assertEquals(mapVertex.getID(), inputEdge2.getSource().getProducer().getJobvertexId()); int inputPartition1 = inputEdge1.getSource().getPartitionNumber(); assertFalse(mapSubpartitions.contains(inputPartition1)); mapSubpartitions.add(inputPartition1); int inputPartition2 = inputEdge2.getSource().getPartitionNumber(); assertFalse(mapSubpartitions.contains(inputPartition2)); mapSubpartitions.add(inputPartition2);
private ExecutionEdge[] connectAllToAll(IntermediateResultPartition[] sourcePartitions, int inputNumber) { ExecutionEdge[] edges = new ExecutionEdge[sourcePartitions.length]; for (int i = 0; i < sourcePartitions.length; i++) { IntermediateResultPartition irp = sourcePartitions[i]; edges[i] = new ExecutionEdge(irp, this, inputNumber); } return edges; }
ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex(); int maxParallelism = vertex.getMaxParallelism(); producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling)); int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size(); IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult(); final IntermediateDataSetID resultId = consumedIntermediateResult.getId(); final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex(); int maxParallelism = vertex.getMaxParallelism(); producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling)); int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size(); IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult(); final IntermediateDataSetID resultId = consumedIntermediateResult.getId(); final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
CompletableFuture<TaskManagerLocation> locationFuture = sources[k].getSource().getProducer().getCurrentTaskManagerLocationFuture();
private ExecutionEdge[] connectAllToAll(IntermediateResultPartition[] sourcePartitions, int inputNumber) { ExecutionEdge[] edges = new ExecutionEdge[sourcePartitions.length]; for (int i = 0; i < sourcePartitions.length; i++) { IntermediateResultPartition irp = sourcePartitions[i]; edges[i] = new ExecutionEdge(irp, this, inputNumber); } return edges; }
ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex(); int maxParallelism = vertex.getMaxParallelism(); producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling)); int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size(); IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult(); final IntermediateDataSetID resultId = consumedIntermediateResult.getId(); final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
SimpleSlot sourceSlot = sources[k].getSource().getProducer().getCurrentAssignedResource(); if (sourceSlot != null) {
private ExecutionEdge[] connectAllToAll(IntermediateResultPartition[] sourcePartitions, int inputNumber) { ExecutionEdge[] edges = new ExecutionEdge[sourcePartitions.length]; for (int i = 0; i < sourcePartitions.length; i++) { IntermediateResultPartition irp = sourcePartitions[i]; edges[i] = new ExecutionEdge(irp, this, inputNumber); } return edges; }
ExecutionJobVertex vertex = consumer.get(0).getTarget().getJobVertex(); int maxParallelism = vertex.getMaxParallelism(); producedPartitions.add(ResultPartitionDeploymentDescriptor.from(partition, maxParallelism, lazyScheduling)); int numConsumerEdges = edges[0].getSource().getConsumers().get(0).size(); IntermediateResult consumedIntermediateResult = edges[0].getSource().getIntermediateResult(); final IntermediateDataSetID resultId = consumedIntermediateResult.getId(); final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
CompletableFuture<TaskManagerLocation> locationFuture = sources[k].getSource().getProducer().getCurrentTaskManagerLocationFuture();
return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[subTaskIndex], this, inputNumber) }; return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[sourcePartition], this, inputNumber) }; edges[i] = new ExecutionEdge(sourcePartitions[startIndex + i], this, inputNumber); edges[i] = new ExecutionEdge(sourcePartitions[start + i], this, inputNumber);
CompletableFuture<TaskManagerLocation> locationFuture = sources[k].getSource().getProducer().getCurrentTaskManagerLocationFuture();
return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[subTaskIndex], this, inputNumber) }; return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[sourcePartition], this, inputNumber) }; edges[i] = new ExecutionEdge(sourcePartitions[startIndex + i], this, inputNumber); edges[i] = new ExecutionEdge(sourcePartitions[start + i], this, inputNumber);
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) { final DistributionPattern pattern = edge.getDistributionPattern(); final IntermediateResultPartition[] sourcePartitions = source.getPartitions(); ExecutionEdge[] edges; switch (pattern) { case POINTWISE: edges = connectPointwise(sourcePartitions, inputNumber); break; case ALL_TO_ALL: edges = connectAllToAll(sourcePartitions, inputNumber); break; default: throw new RuntimeException("Unrecognized distribution pattern."); } this.inputEdges[inputNumber] = edges; // add the consumers to the source // for now (until the receiver initiated handshake is in place), we need to register the // edges as the execution graph for (ExecutionEdge ee : edges) { ee.getSource().addConsumer(ee, consumerNumber); } }
return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[subTaskIndex], this, inputNumber) }; return new ExecutionEdge[] { new ExecutionEdge(sourcePartitions[sourcePartition], this, inputNumber) }; edges[i] = new ExecutionEdge(sourcePartitions[startIndex + i], this, inputNumber); edges[i] = new ExecutionEdge(sourcePartitions[start + i], this, inputNumber);