@JsonCreator public static Lifespan jsonCreator(String value) { if (value.equals("TaskWide")) { return Lifespan.taskWide(); } checkArgument(value.startsWith("Group")); return Lifespan.driverGroup(parseInt(value.substring("Group".length()))); }
public static Lifespan driverGroup(int id) { return new Lifespan(true, id); }
private ConnectorPartitionHandle partitionHandleFor(Lifespan lifespan) { if (lifespan.isTaskWide()) { return NOT_PARTITIONED; } return partitionHandles.get(lifespan.getId()); }
@JsonCreator public Split( @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorSplit") ConnectorSplit connectorSplit, @JsonProperty("lifespan") Lifespan lifespan) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); this.connectorSplit = requireNonNull(connectorSplit, "connectorSplit is null"); this.lifespan = requireNonNull(lifespan, "lifespan is null"); }
Node nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
@Override public void scheduleInitial(SourceScheduler scheduler) { checkState(!initialScheduled); initialScheduled = true; int driverGroupsScheduledPerTask = 0; while (driverGroups.hasNext()) { for (int i = 0; i < allNodes.size() && driverGroups.hasNext(); i++) { int driverGroupId = driverGroups.nextInt(); checkState(!bucketNodeMap.getAssignedNode(driverGroupId).isPresent()); bucketNodeMap.assignBucketToNode(driverGroupId, allNodes.get(i)); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); } driverGroupsScheduledPerTask++; if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduledPerTask == concurrentLifespansPerTask.getAsInt()) { break; } } if (!driverGroups.hasNext()) { scheduler.noMoreLifespans(); } }
private void checkLifespan(PipelineExecutionStrategy executionStrategy, Lifespan lifespan) { switch (executionStrategy) { case GROUPED_EXECUTION: checkArgument(!lifespan.isTaskWide(), "Expect driver-group life cycle for grouped ExecutionStrategy. Got task-wide life cycle."); break; case UNGROUPED_EXECUTION: checkArgument(lifespan.isTaskWide(), "Expect task-wide life cycle for ungrouped ExecutionStrategy. Got driver-group life cycle."); break; default: throw new IllegalArgumentException("Unknown executionStrategy: " + executionStrategy); } }
@JsonCreator public Split( @JsonProperty("connectorId") ConnectorId connectorId, @JsonProperty("transactionHandle") ConnectorTransactionHandle transactionHandle, @JsonProperty("connectorSplit") ConnectorSplit connectorSplit, @JsonProperty("lifespan") Lifespan lifespan) { this.connectorId = requireNonNull(connectorId, "connectorId is null"); this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null"); this.connectorSplit = requireNonNull(connectorSplit, "connectorSplit is null"); this.lifespan = requireNonNull(lifespan, "lifespan is null"); }
Node nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new); bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
@Override public void scheduleInitial(SourceScheduler scheduler) { checkState(!initialScheduled); initialScheduled = true; int driverGroupsScheduledPerTask = 0; while (driverGroups.hasNext()) { for (int i = 0; i < allNodes.size() && driverGroups.hasNext(); i++) { int driverGroupId = driverGroups.nextInt(); checkState(!bucketNodeMap.getAssignedNode(driverGroupId).isPresent()); bucketNodeMap.assignBucketToNode(driverGroupId, allNodes.get(i)); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); } driverGroupsScheduledPerTask++; if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduledPerTask == concurrentLifespansPerTask.getAsInt()) { break; } } if (!driverGroups.hasNext()) { scheduler.noMoreLifespans(); } }
private ConnectorPartitionHandle partitionHandleFor(Lifespan lifespan) { if (lifespan.isTaskWide()) { return NOT_PARTITIONED; } return partitionHandles.get(lifespan.getId()); }
public void addCompletedDriverGroup(Lifespan driverGroup) { checkArgument(!driverGroup.isTaskWide(), "driverGroup is task-wide, not a driver group."); completedDriverGroups.add(driverGroup); }
@JsonCreator public static Lifespan jsonCreator(String value) { if (value.equals("TaskWide")) { return Lifespan.taskWide(); } checkArgument(value.startsWith("Group")); return Lifespan.driverGroup(parseInt(value.substring("Group".length()))); }
public TaskWideInternalJoinBridgeDataManager(Function<Lifespan, T> lookupSourceFactoryProvider, int probeFactoryCount, int outerFactoryCount) { joinBridge = lookupSourceFactoryProvider.apply(Lifespan.taskWide()); joinLifecycle = new JoinLifecycle(joinBridge, probeFactoryCount, outerFactoryCount); }
IntListIterator driverGroupsIterator = nodeToDriverGroupsMap.get(driverGroupToNodeMap.get(driverGroup.getId())); if (!driverGroupsIterator.hasNext()) { continue; scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); totalDriverGroupsScheduled++;
public void scheduleInitial(SourceScheduler scheduler) { checkState(!initialScheduled); initialScheduled = true; for (Map.Entry<Node, IntListIterator> entry : nodeToDriverGroupsMap.entrySet()) { IntListIterator driverGroupsIterator = entry.getValue(); int driverGroupsScheduled = 0; while (driverGroupsIterator.hasNext()) { int driverGroupId = driverGroupsIterator.nextInt(); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); totalDriverGroupsScheduled++; driverGroupsScheduled++; if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduled == concurrentLifespansPerTask.getAsInt()) { break; } } } verify(totalDriverGroupsScheduled <= driverGroupToNodeMap.size()); if (totalDriverGroupsScheduled == driverGroupToNodeMap.size()) { scheduler.noMoreLifespans(); } }
private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) { ConnectorNodePartitioningProvider partitioningProvider = partitioningProviders.get(partitioningHandle.getConnectorId().get()); checkArgument(partitioningProvider != null, "No partitioning provider for connector %s", partitioningHandle.getConnectorId().get()); ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction( partitioningHandle.getTransactionHandle().orElse(null), session.toConnectorSession(), partitioningHandle.getConnectorHandle()); checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle); return split -> { int bucket; if (split.getConnectorSplit() instanceof EmptySplit) { bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId(); } else { bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit()); } if (!split.getLifespan().isTaskWide()) { checkArgument(split.getLifespan().getId() == bucket); } return bucket; }; }
public void addCompletedDriverGroup(Lifespan driverGroup) { checkArgument(!driverGroup.isTaskWide(), "driverGroup is task-wide, not a driver group."); completedDriverGroups.add(driverGroup); }