@Override public void execute(final ExecutionStage stage, OptimizationContext optimizationContext, ExecutionState executionState) { Queue<ExecutionTask> scheduledTasks = new LinkedList<>(stage.getStartTasks()); Set<ExecutionTask> executedTasks = new HashSet<>(); while (!scheduledTasks.isEmpty()) { final ExecutionTask task = scheduledTasks.poll(); if (executedTasks.contains(task)) continue; this.execute(task, optimizationContext, executionState); executedTasks.add(task); Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == stage) .forEach(scheduledTasks::add); } }
/** * Creates a new, hierarchical instance. Mimes the {@code original}'s properties except for the {@link #consumers}. * * @param original the original instance whose properties will be mimed */ protected Channel(Channel original) { this.descriptor = original.getDescriptor(); this.original = original.getOriginal(); assert this.original == null || !this.original.isCopy(); this.producer = original.getProducer(); this.producerSlot = original.getProducerSlot(); }
/** * Exchanges input {@link Channel}. Will also update the {@link Channel}'s consumers appropriately. */ public void exchangeInputChannel(Channel currentChannel, Channel newChannel) { for (int inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == currentChannel) { currentChannel.getConsumers().remove(this); this.setInputChannel(inputIndex, null); newChannel.addConsumer(this, inputIndex); return; } } throw new IllegalArgumentException(String.format("%s is not an input of %s.", currentChannel, this)); }
/** * Copies the siblings of the given {@code channel} into this instance. */ private void adoptSiblings(Channel channel) { for (Channel newSibling : channel.siblings) { this.addSibling(newSibling); } channel.removeSiblings(); }
/** * Merges this instance into the original instance ({@link #getOriginal()}. * <p>The consumers of the original instance are cleared and replaced with the consumers of this instance. * For all other properties, the original and this instance should agree.</p> */ public void mergeIntoOriginal() { if (!this.isCopy()) return; this.getOriginal().copyConsumersFrom(this); this.getOriginal().adoptSiblings(this); }
/** * The given instance should build upon the open {@link Channel}s of this instance. Then, this instance will be * expanded with the content of the given instance. * * @param expansion extends this instance, but they are not overlapping */ public void expand(ExecutionPlan expansion) { for (Channel openChannel : expansion.getOpenInputChannels()) { openChannel.mergeIntoOriginal(); final Channel original = openChannel.getOriginal(); final ExecutionStage producerStage = original.getProducer().getStage(); assert producerStage != null : String.format("No stage found for %s.", original.getProducer()); for (ExecutionTask consumer : original.getConsumers()) { final ExecutionStage consumerStage = consumer.getStage(); assert consumerStage != null : String.format("No stage found for %s.", consumer); // Equal stages possible on "partially open" Channels. if (producerStage != consumerStage) { producerStage.addSuccessor(consumerStage); } } } }
/** * Find {@link ChannelDescriptor}s of {@link Channel}s that can be reached from the current * {@link ChannelDescriptor}. This is relevant only when there are {@link #existingChannels}. * * @param descriptor from which successor {@link ChannelDescriptor}s are requested * @return the successor {@link ChannelDescriptor}s or {@code null} if no restrictions apply */ private Set<ChannelDescriptor> getSuccessorChannelDescriptors(ChannelDescriptor descriptor) { final Channel channel = this.existingChannels.get(descriptor); if (channel == null || this.openChannelDescriptors.contains(descriptor)) return null; Set<ChannelDescriptor> result = new HashSet<>(); for (ExecutionTask consumer : channel.getConsumers()) { if (!consumer.getOperator().isAuxiliary()) continue; for (Channel successorChannel : consumer.getOutputChannels()) { result.add(successorChannel.getDescriptor()); } } return result; }
@Override public String toString() { return String.format("%s[%s->%s]", this.getClass().getSimpleName(), this.getProducer() == null ? this.getProducerSlot() : this.getProducer(), this.getConsumers()); }
/** * Checks whether the given {@link Channel} is inside of a {@link ExecutionStageLoop}. * * @param channel the said {@link Channel} * @return whether the {@link Channel} is in a {@link ExecutionStageLoop} */ private static boolean checkIfIsInLoopChannel(Channel channel) { final ExecutionStageLoop producerLoop = channel.getProducer().getStage().getLoop(); return producerLoop != null && channel.getConsumers().stream().anyMatch( consumer -> consumer.getStage().getLoop() == producerLoop ); }
if (!this.existingDestinationChannelIndices.get(index) && this.openChannelDescriptors.contains(baseChannel.getDescriptor())) { if (baseChannelCopy == null) baseChannelCopy = baseChannel.copy(); junction.setTargetChannel(index, baseChannelCopy); } else { baseChannelCopy = this.openChannelDescriptors.contains(baseChannel.getDescriptor()) ? baseChannel.copy() : baseChannel; final ExecutionTask producer = newChannel.getProducer(); final ExecutionOperator conversionOperator = producer.getOperator(); conversionOperator.setName(String.format(
@Override public FileChannel.Descriptor getDescriptor() { return (FileChannel.Descriptor) super.getDescriptor(); }
/** * Copies the consumers of the given {@code channel} into this instance. */ private void copyConsumersFrom(Channel channel) { assert channel.getOriginal() == this; for (ExecutionTask consumer : new ArrayList<>(channel.getConsumers())) { // We must take care not to copy back channels, that we already have in the original. assert this.consumers.stream() .noneMatch(existingConsumer -> existingConsumer.getOperator().equals(consumer.getOperator())) : String.format("Conflict when copying consumers from %s (%s) to %s (%s).", this, this.consumers, channel, channel.getConsumers() ); consumer.exchangeInputChannel(channel, this); } }
/** * Handle the upstream neighbors of the given {@code task} by expanding the {@code expandableStage}. */ private void expandDownstream(ExecutionTask task, InterimStage expandableStage) { for (Channel channel : task.getOutputChannels()) { assert channel != null : String.format("%s has null output channels.", task); if (channel.isExecutionBreaker()) { expandableStage.setOutbound(task); } for (ExecutionTask consumer : channel.getConsumers()) { final InterimStage assignedStage = this.assignedInterimStages.get(consumer); if (assignedStage == null) { this.handleTaskWithoutPlatformExecution(consumer, /*channel.isExecutionBreaker() ? null : */ expandableStage); } } } }
/** * Put new {@link ChannelInstance}s to the {@link #executionState} and release input {@link ChannelInstance}s. */ private void updateExecutionState() { for (final ChannelInstance channelInstance : this.allChannelInstances) { // Capture outbound ChannelInstances. if (channelInstance.getChannel().isBetweenStages() || channelInstance.getChannel().getConsumers().stream() .anyMatch(consumer -> consumer.getOperator().isLoopHead())) { this.executionState.register(channelInstance); } // Release the ChannelInstance. channelInstance.noteDiscardedReference(true); } this.allChannelInstances.clear(); } }
Integer thisOutIndex = channel.getProducerSlot()==null ? 0 : channel.getProducerSlot().getIndex(); jsonConnectsTo.put(thisOutIndex.toString(), perOutputThatList); for (ExecutionTask consumer : channel.getConsumers()) { HashMap<String, Object> jsonThatOp = new HashMap<>(); jsonThatOp.put(consumer.getOperator().getName(),
while (existingChannel.getProducerSlot() != sourceOutput) { existingChannel = OptimizationUtils.getPredecessorChannel(existingChannel); this.sourceChannelDescriptor = sourceChannel.getDescriptor(); this.openChannelDescriptors = new HashSet<>(openChannels.size()); for (Channel openChannel : openChannels) { this.openChannelDescriptors.add(openChannel.getDescriptor()); for (Channel existingDestinationChannel : this.existingDestinationChannels.values()) { final Bitmask channelIndices = this.kernelDestChannelDescriptorsToIndices .get(existingDestinationChannel.getDescriptor()) .and(this.existingDestinationChannelIndices); while (true) { this.reachableExistingDestinationChannelIndices.compute( existingDestinationChannel.getDescriptor(), (k, v) -> v == null ? new Bitmask(channelIndices) : v.orInPlace(channelIndices) ); if (existingDestinationChannel.getDescriptor().equals(this.sourceChannelDescriptor)) break; existingDestinationChannel = OptimizationUtils.getPredecessorChannel(existingDestinationChannel);
/** * Create output {@link ChannelInstance}s for this instance. * * @param task the {@link ExecutionTask} in which this instance is being wrapped * @param producerOperatorContext the {@link OptimizationContext.OperatorContext} for this instance * @param inputChannelInstances the input {@link ChannelInstance}s for the {@code task} * @return */ default ChannelInstance[] createOutputChannelInstances(Executor executor, ExecutionTask task, OptimizationContext.OperatorContext producerOperatorContext, List<ChannelInstance> inputChannelInstances) { assert task.getOperator() == this; ChannelInstance[] channelInstances = new ChannelInstance[task.getNumOuputChannels()]; for (int outputIndex = 0; outputIndex < channelInstances.length; outputIndex++) { final Channel outputChannel = task.getOutputChannel(outputIndex); final ChannelInstance outputChannelInstance = outputChannel.createInstance(executor, producerOperatorContext, outputIndex); channelInstances[outputIndex] = outputChannelInstance; } return channelInstances; }
@Override public Channel convert(Channel sourceChannel, Configuration configuration, Collection<OptimizationContext> optimizationContexts, CardinalityEstimate optCardinality) { // Create the ExecutionOperator. final ExecutionOperator executionOperator = this.executionOperatorFactory.apply(sourceChannel, configuration); assert executionOperator.getNumInputs() <= 1 && executionOperator.getNumOutputs() <= 1; executionOperator.setAuxiliary(true); // Set up the Channels and the ExecutionTask. final ExecutionTask task = new ExecutionTask(executionOperator, 1, 1); sourceChannel.addConsumer(task, 0); final Channel outputChannel = task.initializeOutputChannel(0, configuration); sourceChannel.addSibling(outputChannel); setCardinalityAndTimeEstimates(sourceChannel, optimizationContexts, optCardinality, task); return outputChannel; }
private void connectToSuccessorTasks(int outputIndex, Platform platform, Collection<Activation> collector) { final OutputSlot<?> output = this.operator.getOutput(outputIndex); // TODO: Make generic: There might be multiple OutputSlots for final loop outputs (one for each iteration). final Junction junction = this.getJunction(output); LoggerFactory.getLogger(this.getClass()).debug("Connecting {} -> {}.", output, junction); assert junction != null : String.format("No junction found for %s.", output); this.executionTask.setOutputChannel(outputIndex, junction.getSourceChannel()); for (int targetIndex = 0; targetIndex < junction.getNumTargets(); targetIndex++) { final Channel targetChannel = junction.getTargetChannel(targetIndex); final InputSlot<?> targetInput = junction.getTargetInput(targetIndex); final ExecutionTask successorTask = ExecutionTaskFlowCompiler.this.getOrCreateExecutionTask((ExecutionOperator) targetInput.getOwner()); targetChannel.addConsumer(successorTask, targetInput.getIndex()); this.createActivation(targetInput.unchecked(), collector); } }
nextNodeId++, inputChannel.getClass().getCanonicalName(), inputChannel.getDataSetType().getDataUnitType().getTypeClass().getName() ); channelNodeMap.put(inputChannel, channelNode); nextNodeId++, outputChannel.getClass().getCanonicalName(), outputChannel.getDataSetType().getDataUnitType().getTypeClass().getName() ); channelNodeMap.put(outputChannel, channelNode);