private void checkLifespan(PipelineExecutionStrategy executionStrategy, Lifespan lifespan) { switch (executionStrategy) { case GROUPED_EXECUTION: checkArgument(!lifespan.isTaskWide(), "Expect driver-group life cycle for grouped ExecutionStrategy. Got task-wide life cycle."); break; case UNGROUPED_EXECUTION: checkArgument(lifespan.isTaskWide(), "Expect task-wide life cycle for ungrouped ExecutionStrategy. Got driver-group life cycle."); break; default: throw new IllegalArgumentException("Unknown executionStrategy: " + executionStrategy); } }
public void addCompletedDriverGroup(Lifespan driverGroup) { checkArgument(!driverGroup.isTaskWide(), "driverGroup is task-wide, not a driver group."); completedDriverGroups.add(driverGroup); }
public void onLifespanFinished(Iterable<Lifespan> newlyCompletedDriverGroups) { checkState(initialScheduled); SettableFuture<?> newDriverGroupReady; synchronized (this) { for (Lifespan newlyCompletedDriverGroup : newlyCompletedDriverGroups) { checkArgument(!newlyCompletedDriverGroup.isTaskWide()); recentlyCompletedDriverGroups.add(newlyCompletedDriverGroup); } newDriverGroupReady = this.newDriverGroupReady; } newDriverGroupReady.set(null); }
@Override public void onLifespanFinished(Iterable<Lifespan> newlyCompletedDriverGroups) { checkState(initialScheduled); SettableFuture<?> newDriverGroupReady; synchronized (this) { for (Lifespan newlyCompletedDriverGroup : newlyCompletedDriverGroups) { checkArgument(!newlyCompletedDriverGroup.isTaskWide()); recentlyCompletedDriverGroups.add(newlyCompletedDriverGroup); } newDriverGroupReady = this.newDriverGroupReady; } newDriverGroupReady.set(null); }
private ConnectorPartitionHandle partitionHandleFor(Lifespan lifespan) { if (lifespan.isTaskWide()) { return NOT_PARTITIONED; } return partitionHandles.get(lifespan.getId()); }
public synchronized LocalExchange getLocalExchange(Lifespan lifespan) { if (exchangeSourcePipelineExecutionStrategy == UNGROUPED_EXECUTION) { checkArgument(lifespan.isTaskWide(), "LocalExchangeFactory is declared as UNGROUPED_EXECUTION. Driver-group exchange cannot be created."); } else { checkArgument(!lifespan.isTaskWide(), "LocalExchangeFactory is declared as GROUPED_EXECUTION. Task-wide exchange cannot be created."); } return localExchangeMap.computeIfAbsent(lifespan, ignored -> { checkState(noMoreSinkFactories); LocalExchange localExchange = new LocalExchange(numSinkFactories, bufferCount, partitioning, types, partitionChannels, partitionHashChannel, maxBufferedBytes); for (LocalExchangeSinkFactoryId closedSinkFactoryId : closedSinkFactories) { localExchange.getSinkFactory(closedSinkFactoryId).close(); } return localExchange; }); }
public synchronized boolean isNoMoreDriverRunners(Lifespan lifespan) { if (!lifespan.isTaskWide()) { return per(lifespan).pipelinesWithNoMoreDriverRunners == pipelineWithDriverGroupLifeCycleCount; } else { return per(lifespan).pipelinesWithNoMoreDriverRunners == pipelineWithTaskLifeCycleCount; } }
private void scheduleDriversForDriverGroupLifeCycle(Lifespan lifespan) { // This method is called when a split that belongs to a previously unseen driver group is scheduled. // It schedules drivers for all the pipelines that have driver group life cycle. if (lifespan.isTaskWide()) { checkArgument(driverRunnerFactoriesWithDriverGroupLifeCycle.isEmpty(), "Instantiating pipeline of driver group lifecycle at task level is not allowed"); return; } List<DriverSplitRunner> runners = new ArrayList<>(); for (DriverSplitRunnerFactory driverSplitRunnerFactory : driverRunnerFactoriesWithDriverGroupLifeCycle) { for (int i = 0; i < driverSplitRunnerFactory.getDriverInstances().orElse(1); i++) { runners.add(driverSplitRunnerFactory.createDriverRunner(null, lifespan)); } } enqueueDriverSplitRunner(true, runners); for (DriverSplitRunnerFactory driverRunnerFactory : driverRunnerFactoriesWithDriverGroupLifeCycle) { driverRunnerFactory.noMoreDriverRunner(ImmutableList.of(lifespan)); } }
private void checkLifespanCompletion(Lifespan lifespan) { if (lifespan.isTaskWide()) { return; // not a driver group } if (!isNoMoreDriverRunners(lifespan)) { return; } if (getRemainingDriver(lifespan) != 0) { return; } taskContext.addCompletedDriverGroup(lifespan); }
public Optional<PlanNodeId> getSchedulingPlanNode() { checkState(!isDone()); while (!isDone()) { // Return current plan node if this lifespan is compatible with the plan node. // i.e. One of the following bullet points is true: // * The execution strategy of the plan node is grouped. And lifespan represents a driver group. // * The execution strategy of the plan node is ungrouped. And lifespan is task wide. if (manager.stageExecutionDescriptor.isScanGroupedExecution(manager.sourceStartOrder.get(schedulingPlanNodeOrdinal)) != lifespan.isTaskWide()) { return Optional.of(manager.sourceStartOrder.get(schedulingPlanNodeOrdinal)); } // This lifespan is incompatible with the plan node. As a result, this method should either // return empty to indicate that scheduling for this lifespan is blocked, or skip the current // plan node and went on to the next one. Which one of the two happens is dependent on whether // the current plan node has finished scheduling in any other lifespan. // If so, the lifespan can advance to the next plan node. // If not, it should not advance because doing so would violate scheduling order. if (manager.getMaxScheduledPlanNodeOrdinal() == schedulingPlanNodeOrdinal) { return Optional.empty(); } verify(manager.getMaxScheduledPlanNodeOrdinal() > schedulingPlanNodeOrdinal); nextPlanNode(); } return Optional.empty(); }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
if (pendingSplits.isEmpty() && scheduleGroup.state == ScheduleGroupState.NO_MORE_SPLITS) { scheduleGroup.state = ScheduleGroupState.DONE; if (!lifespan.isTaskWide()) { Node node = ((BucketedSplitPlacementPolicy) splitPlacementPolicy).getNodeForBucket(lifespan.getId()); noMoreSplitsNotification = ImmutableMultimap.of(node, lifespan);
if (!lifespan.isTaskWide() && !schedulingLifespan.getAndSetDriversForDriverGroupLifeCycleScheduled()) { scheduleDriversForDriverGroupLifeCycle(lifespan);