private ExecutionPlan assembleExecutionPlan() { final Map<InterimStage, ExecutionStage> finalStages = new HashMap<>(this.allStages.size()); for (ExecutionTask sinkTask : this.executionTaskFlow.getSinkTasks()) { this.assembleExecutionPlan(finalStages, null, sinkTask, new HashSet<>()); } final ExecutionPlan executionPlan = new ExecutionPlan(); finalStages.values().stream().filter(ExecutionStage::isStartingStage).forEach(executionPlan::addStartingStage); return executionPlan; }
/** * Collects all {@link Channel}s that are copies. We recognize them as {@link Channel}s that are open. */ public Collection<Channel> getOpenInputChannels() { return this.collectAllTasks().stream() .flatMap(task -> Arrays.stream(task.getInputChannels())) .filter(Channel::isCopy) .collect(Collectors.toList()); }
/** * Collects all {@link ExecutionTask}s of this instance. * * @return the {@link ExecutionTask}s */ public Set<ExecutionTask> collectAllTasks() { Set<ExecutionTask> allTasks = new HashSet<>(); for (ExecutionStage stage : this.getStages()) { allTasks.addAll(stage.getAllTasks()); } return allTasks; }
private void logStages(ExecutionPlan executionPlan) { if (this.logger.isInfoEnabled()) { StringBuilder sb = new StringBuilder(); Set<ExecutionStage> seenStages = new HashSet<>(); Queue<ExecutionStage> stagedStages = new LinkedList<>(executionPlan.getStartingStages()); ExecutionStage nextStage; while ((nextStage = stagedStages.poll()) != null) { sb.append(nextStage).append(":\n"); nextStage.getPlanAsString(sb, "* "); nextStage.getSuccessors().stream() .filter(seenStages::add) .forEach(stagedStages::add); } this.logger.info("Current execution plan:\n{}", executionPlan.toExtensiveString()); } }
final Set<Channel> openChannels = executionPlan.retain(completedStages); planImplementation, executionPlan, openChannels, completedStages ); final ExecutionPlan executionPlanExpansion = ExecutionPlan.createFrom(executionTaskFlow, this.stageSplittingCriterion); executionPlan.expand(executionPlanExpansion); assert executionPlan.isSane();
/** * Collects all {@link ExecutionStage}s in this instance. * * @return the {@link ExecutionStage}s */ public Set<ExecutionStage> getStages() { Set<ExecutionStage> seenStages = new HashSet<>(); Queue<ExecutionStage> openStages = new LinkedList<>(this.getStartingStages()); while (!openStages.isEmpty()) { final ExecutionStage stage = openStages.poll(); if (seenStages.add(stage)) { openStages.addAll(stage.getSuccessors()); } } return seenStages; }
/** * Creates a {@link String} representation (not strictly ordered) of this instance. * * @return the {@link String} representation */ public String toExtensiveString() { return this.toExtensiveString(false); }
/** * The given instance should build upon the open {@link Channel}s of this instance. Then, this instance will be * expanded with the content of the given instance. * * @param expansion extends this instance, but they are not overlapping */ public void expand(ExecutionPlan expansion) { for (Channel openChannel : expansion.getOpenInputChannels()) { openChannel.mergeIntoOriginal(); final Channel original = openChannel.getOriginal(); final ExecutionStage producerStage = original.getProducer().getStage(); assert producerStage != null : String.format("No stage found for %s.", original.getProducer()); for (ExecutionTask consumer : original.getConsumers()) { final ExecutionStage consumerStage = consumer.getStage(); assert consumerStage != null : String.format("No stage found for %s.", consumer); // Equal stages possible on "partially open" Channels. if (producerStage != consumerStage) { producerStage.addSuccessor(consumerStage); } } } }
this.optimizationRound.start("Create Initial Execution Plan", "Split Stages"); final ExecutionTaskFlow executionTaskFlow = ExecutionTaskFlow.createFrom(this.planImplementation); final ExecutionPlan executionPlan = ExecutionPlan.createFrom(executionTaskFlow, this.stageSplittingCriterion); this.optimizationRound.stop("Create Initial Execution Plan", "Split Stages");
/** * Sets up a {@link Breakpoint} for an {@link ExecutionPlan}. * * @param executionPlan for that the {@link Breakpoint} should be set * @param round {@link TimeMeasurement} to be extended for any interesting time measurements */ private void setUpBreakpoint(ExecutionPlan executionPlan, TimeMeasurement round) { // Set up appropriate Breakpoints. final TimeMeasurement breakpointRound = round.start("Configure Breakpoint"); FixBreakpoint immediateBreakpoint = new FixBreakpoint(); final Set<ExecutionStage> completedStages = this.crossPlatformExecutor.getCompletedStages(); if (completedStages.isEmpty()) { executionPlan.getStartingStages().forEach(immediateBreakpoint::breakAfter); } else { completedStages.stream() .flatMap(stage -> stage.getSuccessors().stream()) .filter(stage -> !completedStages.contains(stage)) .forEach(immediateBreakpoint::breakAfter); } this.crossPlatformExecutor.setBreakpoint(new ConjunctiveBreakpoint( immediateBreakpoint, this.cardinalityBreakpoint, new NoIterationBreakpoint() // Avoid re-optimization inside of loops. )); breakpointRound.stop(); }
/** * Implements various sanity checks. Problems are logged. */ public boolean isSane() { // 1. Check if every ExecutionTask is assigned an ExecutionStage. final Set<ExecutionTask> allTasks = this.collectAllTasks(); boolean isAllTasksAssigned = allTasks.stream().allMatch(task -> task.getStage() != null); if (!isAllTasksAssigned) { this.logger.error("There are tasks without stages."); } final Set<Channel> allChannels = allTasks.stream() .flatMap(task -> Stream.concat(Arrays.stream(task.getInputChannels()), Arrays.stream(task.getOutputChannels()))) .collect(Collectors.toSet()); boolean isAllChannelsOriginal = allChannels.stream() .allMatch(channel -> !channel.isCopy()); if (!isAllChannelsOriginal) { this.logger.error("There are channels that are copies."); } boolean isAllSiblingsConsistent = true; for (Channel channel : allChannels) { for (Channel sibling : channel.getSiblings()) { if (!allChannels.contains(sibling)) { this.logger.error("A sibling of {}, namely {}, seems to be invalid.", channel, sibling); isAllSiblingsConsistent = false; } } } return isAllTasksAssigned && isAllChannelsOriginal && isAllSiblingsConsistent; }
/** * Clean up {@link ExecutionStage}-related state and re-create {@link StageActivator}s etc. from the {@link ExecutionPlan}/ * * @param executionPlan whose {@link ExecutionStage}s will be executed * @param optimizationContext contains additional optimization info for the {@code executionPlan} */ public void prepare(ExecutionPlan executionPlan, OptimizationContext optimizationContext) { this.allStages.clear(); this.activatedStageActivators.clear(); this.suspendedStages.clear(); // Remove obsolete StageActivators (after re-optimization). this.allStages.addAll(executionPlan.getStages()); new ArrayList<>(this.pendingStageActivators.keySet()).stream() .filter(stage -> !this.allStages.contains(stage)) .forEach(this.pendingStageActivators::remove); // Create StageActivators for all ExecutionStages. for (ExecutionStage stage : this.allStages) { // Avoid re-activating already executed ExecutionStages. if (this.completedStages.contains(stage)) continue; final StageActivator activator = this.getOrCreateActivator( stage, () -> this.determineInitialOptimizationContext(stage, optimizationContext) ); this.tryToActivate(activator); } }
final Set<ExecutionTask> executedTasks = baseplan.collectAllTasks(); executedTasks.forEach(task -> this.executedTasks.put(task.getOperator(), task));
final Set<ExecutionOperator> executedOperators = existingPlan.collectAllTasks().stream() .map(ExecutionTask::getOperator) .collect(Collectors.toSet());
Set<ExecutionTask> executionTasks = executionPlan.collectAllTasks();