private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
@GuardedBy("this") private void mergeIntoPendingSplits(PlanNodeId planNodeId, Set<ScheduledSplit> scheduledSplits, Set<Lifespan> noMoreSplitsForLifespan, boolean noMoreSplits) { checkHoldsLock(); DriverSplitRunnerFactory partitionedDriverFactory = driverRunnerFactoriesWithSplitLifeCycle.get(planNodeId); PendingSplitsForPlanNode pendingSplitsForPlanNode = pendingSplitsByPlanNode.get(planNodeId); partitionedDriverFactory.splitsAdded(scheduledSplits.size()); for (ScheduledSplit scheduledSplit : scheduledSplits) { Lifespan lifespan = scheduledSplit.getSplit().getLifespan(); checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespan); pendingSplitsForPlanNode.getLifespan(lifespan).addSplit(scheduledSplit); schedulingLifespanManager.addLifespanIfAbsent(lifespan); } for (Lifespan lifespanWithNoMoreSplits : noMoreSplitsForLifespan) { checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespanWithNoMoreSplits); pendingSplitsForPlanNode.getLifespan(lifespanWithNoMoreSplits).noMoreSplits(); schedulingLifespanManager.addLifespanIfAbsent(lifespanWithNoMoreSplits); } if (noMoreSplits) { pendingSplitsForPlanNode.setNoMoreSplits(); } }
@GuardedBy("this") private void mergeIntoPendingSplits(PlanNodeId planNodeId, Set<ScheduledSplit> scheduledSplits, Set<Lifespan> noMoreSplitsForLifespan, boolean noMoreSplits) { checkHoldsLock(); DriverSplitRunnerFactory partitionedDriverFactory = driverRunnerFactoriesWithSplitLifeCycle.get(planNodeId); PendingSplitsForPlanNode pendingSplitsForPlanNode = pendingSplitsByPlanNode.get(planNodeId); partitionedDriverFactory.splitsAdded(scheduledSplits.size()); for (ScheduledSplit scheduledSplit : scheduledSplits) { Lifespan lifespan = scheduledSplit.getSplit().getLifespan(); checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespan); pendingSplitsForPlanNode.getLifespan(lifespan).addSplit(scheduledSplit); schedulingLifespanManager.addLifespanIfAbsent(lifespan); } for (Lifespan lifespanWithNoMoreSplits : noMoreSplitsForLifespan) { checkLifespan(partitionedDriverFactory.getPipelineExecutionStrategy(), lifespanWithNoMoreSplits); pendingSplitsForPlanNode.getLifespan(lifespanWithNoMoreSplits).noMoreSplits(); schedulingLifespanManager.addLifespanIfAbsent(lifespanWithNoMoreSplits); } if (noMoreSplits) { pendingSplitsForPlanNode.setNoMoreSplits(); } }