protected ExecutionOperator getOperator() { return this.task.getOperator(); }
/** * Determine the producing {@link OutputSlot} of this {@link Channel} that lies within a {@link RheemPlan}. * We follow non-RheemPlan {@link ExecutionOperator}s because they should merely forward data. */ public static OutputSlot<?> findRheemPlanOutputSlotFor(Channel openChannel) { OutputSlot<?> producerOutput = null; Channel tracedChannel = openChannel; do { final ExecutionTask producer = tracedChannel.getProducer(); final ExecutionOperator producerOperator = producer.getOperator(); if (!producerOperator.isAuxiliary()) { producerOutput = producer.getOutputSlotFor(tracedChannel); } else { assert producer.getNumInputChannels() == 1; tracedChannel = producer.getInputChannel(0); } } while (producerOutput == null); return producerOutput; }
/** * Create output {@link ChannelInstance}s for this instance. * * @param task the {@link ExecutionTask} in which this instance is being wrapped * @param producerOperatorContext the {@link OptimizationContext.OperatorContext} for this instance * @param inputChannelInstances the input {@link ChannelInstance}s for the {@code task} * @return */ default ChannelInstance[] createOutputChannelInstances(Executor executor, ExecutionTask task, OptimizationContext.OperatorContext producerOperatorContext, List<ChannelInstance> inputChannelInstances) { assert task.getOperator() == this; ChannelInstance[] channelInstances = new ChannelInstance[task.getNumOuputChannels()]; for (int outputIndex = 0; outputIndex < channelInstances.length; outputIndex++) { final Channel outputChannel = task.getOutputChannel(outputIndex); final ChannelInstance outputChannelInstance = outputChannel.createInstance(executor, producerOperatorContext, outputIndex); channelInstances[outputIndex] = outputChannelInstance; } return channelInstances; }
/** * Sets an input {@link Channel} for this instance. Consider using {@link Channel#addConsumer(ExecutionTask, int)} * instead. */ void setInputChannel(int index, Channel channel) { assert channel == null || this.getInputChannel(index) == null : String.format("Cannot set up %s for %s@%d: There is already %s.", channel, this.getOperator(), index, this.getInputChannel(index)); this.getInputChannels()[index] = channel; }
/** * Sets an output {@link Channel} for this instance. */ public void setOutputChannel(int index, Channel channel) { assert this.getOutputChannel(index) == null : String.format("Output channel %d of %s is already set to %s.", index, this, this.getOutputChannel(index)); this.getOutputChannels()[index] = channel; channel.setProducer(this); }
/** * Finds the single input {@link Channel} of the given {@code channel}'s producing {@link ExecutionTask}. * * @param channel whose predecessor is requested * @return the preceeding {@link Channel} */ public static Channel getPredecessorChannel(Channel channel) { final ExecutionTask producer = channel.getProducer(); assert producer != null && producer.getNumInputChannels() == 1; return producer.getInputChannel(0); }
/** * Brings the given {@code task} into execution. */ private void execute(ExecutionTask task, OptimizationContext optimizationContext, ExecutionState executionState) { final GraphChiExecutionOperator graphChiExecutionOperator = (GraphChiExecutionOperator) task.getOperator(); ChannelInstance[] inputChannelInstances = new ChannelInstance[task.getNumInputChannels()]; for (int i = 0; i < inputChannelInstances.length; i++) { inputChannelInstances[i] = executionState.getChannelInstance(task.getInputChannel(i)); } final OptimizationContext.OperatorContext operatorContext = optimizationContext.getOperatorContext(graphChiExecutionOperator); ChannelInstance[] outputChannelInstances = new ChannelInstance[task.getNumOuputChannels()]; for (int i = 0; i < outputChannelInstances.length; i++) { outputChannelInstances[i] = task.getOutputChannel(i).createInstance(this, operatorContext, i); } long startTime = System.currentTimeMillis(); final Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> results = graphChiExecutionOperator.execute(inputChannelInstances, outputChannelInstances, operatorContext); long endTime = System.currentTimeMillis(); final Collection<ExecutionLineageNode> executionLineageNodes = results.getField0(); final Collection<ChannelInstance> producedChannelInstances = results.getField1(); for (ChannelInstance outputChannelInstance : outputChannelInstances) { if (outputChannelInstance != null) { executionState.register(outputChannelInstance); } } final PartialExecution partialExecution = this.createPartialExecution(executionLineageNodes, endTime - startTime); executionState.add(partialExecution); this.registerMeasuredCardinalities(producedChannelInstances); }
/** * Determine the consuming {@link InputSlot}s of the given {@link Channel} that lie within a {@link RheemPlan} and * have not been executed yet. * We follow non-RheemPlan {@link ExecutionOperator}s because they should merely forward data. */ private Collection<InputSlot<?>> findRheemPlanInputSlotFor(Channel channel, Set<ExecutionStage> executedStages) { Collection<InputSlot<?>> result = new LinkedList<>(); for (ExecutionTask consumerTask : channel.getConsumers()) { if (executedStages.contains(consumerTask.getStage())) continue; if (!consumerTask.getOperator().isAuxiliary()) { result.add(consumerTask.getInputSlotFor(channel)); } else { for (Channel consumerOutputChannel : consumerTask.getOutputChannels()) { result.addAll(this.findRheemPlanInputSlotFor(consumerOutputChannel, executedStages)); } } } return result; }
/** * Returns the {@link InputSlot} of the {@link ExecutionOperator} that is associated to the given {@link Channel}. * * @return the {@link InputSlot} or {@code null} if none */ public InputSlot<?> getInputSlotFor(Channel channel) { // Simple implementation: linear search. for (int inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == channel) { return inputIndex < this.getOperator().getNumInputs() ? this.getOperator().getInput(inputIndex) : null; } } throw new IllegalArgumentException(String.format("%s does not belong to %s.", channel, this)); }
/** * Checks if the given {@code channel} is a feedback to {@code task} (i.e., it closes a data flow cycle). */ private boolean checkIfFeedbackChannel(ExecutionTask task, Channel channel) { if (!task.getOperator().isLoopHead()) return false; final InputSlot<?> input = task.getInputSlotFor(channel); return input != null && input.isFeedback(); }
/** * Starts building an {@link InterimStage} starting from the given {@link ExecutionTask} unless there is one already. * If a {@link PlatformExecution} is provided, the {@link InterimStage} will be associated with it. */ private void createStageFor(ExecutionTask task, PlatformExecution platformExecution) { assert task.getStage() == null : String.format("%s has already stage %s.", task, task.getStage()); // See if there is already an InterimStage. if (this.assignedInterimStages.containsKey(task)) { return; } // Create a new PlatformExecution if none. if (platformExecution == null) { Platform platform = task.getOperator().getPlatform(); platformExecution = new PlatformExecution(platform); } // Create the InterimStage and expand it. InterimStage initialStage = new InterimStageImpl(platformExecution); this.addStage(initialStage); this.assignTaskAndExpand(task, initialStage); }
/** * Traverse the given {@link Channel} downstream and collect any {@link Channel} that feeds some * {@link InputSlot} of a non-conversion {@link ExecutionOperator}. * * @param channel the {@link Channel} to traverse from * @see #existingChannels * @see #existingDestinationChannels */ private void collectExistingChannels(Channel channel) { this.existingChannels.put(channel.getDescriptor(), channel); for (ExecutionTask consumer : channel.getConsumers()) { final ExecutionOperator operator = consumer.getOperator(); if (!operator.isAuxiliary()) { final InputSlot<?> input = consumer.getInputSlotFor(channel); this.existingDestinationChannels.put(input, channel); int destIndex = 0; while (this.destInputs.get(destIndex) != input) destIndex++; this.existingDestinationChannelIndices.set(destIndex); } else { for (Channel outputChannel : consumer.getOutputChannels()) { if (outputChannel != null) this.collectExistingChannels(outputChannel); } } } }
/** * Removes the given {@link Channel} as output of this instance. * * @return the former output index the {@link Channel} */ public int removeOutputChannel(Channel outputChannel) { int outputIndex; for (outputIndex = 0; outputIndex < this.getNumOuputChannels(); outputIndex++) { if (this.getOutputChannel(outputIndex) == outputChannel) { this.getOutputChannels()[outputIndex] = null; outputChannel.setProducer(null); return outputIndex; } } throw new IllegalArgumentException(String.format("%s is not an output of %s.", outputChannel, this)); }
for (ExecutionTask executionTask : inboundChannel.getConsumers()) { if (executionTask.getStage() != this.stage) continue; if (executionTask.getOperator().isLoopHead()) { LoopHeadOperator loopHead = (LoopHeadOperator) executionTask.getOperator(); final InputSlot<?> targetInput = executionTask.getInputSlotFor(inboundChannel); if (loopHead.getLoopBodyInputs().contains(targetInput)) { this.iterationInboundChannels.add(inboundChannel);
for (ExecutionTask executionTask : executionTasks) { ExecutionOperator operator = executionTask.getOperator(); OperatorNode operatorNode = new OperatorNode( nextNodeId++, for (Channel inputChannel : executionTask.getInputChannels()) { if (inputChannel == null) continue; for (Channel outputChannel : executionTask.getOutputChannels()) { if (outputChannel == null) continue;
/** * Find {@link ChannelDescriptor}s of {@link Channel}s that can be reached from the current * {@link ChannelDescriptor}. This is relevant only when there are {@link #existingChannels}. * * @param descriptor from which successor {@link ChannelDescriptor}s are requested * @return the successor {@link ChannelDescriptor}s or {@code null} if no restrictions apply */ private Set<ChannelDescriptor> getSuccessorChannelDescriptors(ChannelDescriptor descriptor) { final Channel channel = this.existingChannels.get(descriptor); if (channel == null || this.openChannelDescriptors.contains(descriptor)) return null; Set<ChannelDescriptor> result = new HashSet<>(); for (ExecutionTask consumer : channel.getConsumers()) { if (!consumer.getOperator().isAuxiliary()) continue; for (Channel successorChannel : consumer.getOutputChannels()) { result.add(successorChannel.getDescriptor()); } } return result; }
/** * Collects all {@link ExecutionTask}s of this instance. */ public Set<ExecutionTask> getAllTasks() { final Queue<ExecutionTask> nextTasks = new LinkedList<>(this.startTasks); final Set<ExecutionTask> allTasks = new HashSet<>(); while (!nextTasks.isEmpty()) { final ExecutionTask task = nextTasks.poll(); assert task.getStage() == this; if (allTasks.add(task) && !this.terminalTasks.contains(task)) { Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == this) .forEach(nextTasks::add); } } return allTasks; }
/** * Removes the given {@link Channel} as input of this instance. * * @return the former input index the {@link Channel} */ public int removeInputChannel(Channel inputChannel) { int inputIndex; for (inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == inputChannel) { this.getInputChannels()[inputIndex] = null; inputChannel.getConsumers().remove(this); return inputIndex; } } throw new IllegalArgumentException(String.format("%s is not an input of %s.", inputChannel, this)); }
/** * Exchanges input {@link Channel}. Will also update the {@link Channel}'s consumers appropriately. */ public void exchangeInputChannel(Channel currentChannel, Channel newChannel) { for (int inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == currentChannel) { currentChannel.getConsumers().remove(this); this.setInputChannel(inputIndex, null); newChannel.addConsumer(this, inputIndex); return; } } throw new IllegalArgumentException(String.format("%s is not an input of %s.", currentChannel, this)); }
/** * Checks if the given {@code channel} is a feedforward to {@code task} (i.e., it constitutes the beginning of a data flow cycle). */ private boolean checkIfFeedforwardChannel(ExecutionTask task, Channel channel) { if (!task.getOperator().isLoopHead()) return false; final OutputSlot<?> output = task.getOutputSlotFor(channel); return output != null && output.isFeedforward(); } }