@JsonCreator public static Lifespan jsonCreator(String value) { if (value.equals("TaskWide")) { return Lifespan.taskWide(); } checkArgument(value.startsWith("Group")); return Lifespan.driverGroup(parseInt(value.substring("Group".length()))); }
@Override public void scheduleInitial(SourceScheduler scheduler) { checkState(!initialScheduled); initialScheduled = true; int driverGroupsScheduledPerTask = 0; while (driverGroups.hasNext()) { for (int i = 0; i < allNodes.size() && driverGroups.hasNext(); i++) { int driverGroupId = driverGroups.nextInt(); checkState(!bucketNodeMap.getAssignedNode(driverGroupId).isPresent()); bucketNodeMap.assignBucketToNode(driverGroupId, allNodes.get(i)); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); } driverGroupsScheduledPerTask++; if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduledPerTask == concurrentLifespansPerTask.getAsInt()) { break; } } if (!driverGroups.hasNext()) { scheduler.noMoreLifespans(); } }
public void scheduleInitial(SourceScheduler scheduler) { checkState(!initialScheduled); initialScheduled = true; for (Map.Entry<Node, IntListIterator> entry : nodeToDriverGroupsMap.entrySet()) { IntListIterator driverGroupsIterator = entry.getValue(); int driverGroupsScheduled = 0; while (driverGroupsIterator.hasNext()) { int driverGroupId = driverGroupsIterator.nextInt(); scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); totalDriverGroupsScheduled++; driverGroupsScheduled++; if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduled == concurrentLifespansPerTask.getAsInt()) { break; } } } verify(totalDriverGroupsScheduled <= driverGroupToNodeMap.size()); if (totalDriverGroupsScheduled == driverGroupToNodeMap.size()) { scheduler.noMoreLifespans(); } }
private void run(LocalExchangeFactory localExchangeFactory, PipelineExecutionStrategy pipelineExecutionStrategy, Consumer<LocalExchange> test) { switch (pipelineExecutionStrategy) { case UNGROUPED_EXECUTION: test.accept(localExchangeFactory.getLocalExchange(Lifespan.taskWide())); break; case GROUPED_EXECUTION: test.accept(localExchangeFactory.getLocalExchange(Lifespan.driverGroup(1))); test.accept(localExchangeFactory.getLocalExchange(Lifespan.driverGroup(12))); test.accept(localExchangeFactory.getLocalExchange(Lifespan.driverGroup(23))); break; default: throw new IllegalArgumentException("Unknown pipelineExecutionStrategy"); } }
scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId)); totalDriverGroupsScheduled++;
LOCAL_EXCHANGE_MAX_BUFFERED_BYTES); try { ungroupedLocalExchangeFactory.getLocalExchange(Lifespan.driverGroup(3)); fail("expected failure");
public static void assertExpectedDriverStats(DriverStats actual) assertEquals(actual.getLifespan(), Lifespan.driverGroup(21));
TABLE_SCAN_NODE_ID, ImmutableSet.of( newScheduledSplit(0, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(1), 0, 1), newScheduledSplit(1, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 100000, 10)), ImmutableSet.of(Lifespan.driverGroup(1)), false))); waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT); ImmutableSet.of(newScheduledSplit(2, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 200000, 300)), ImmutableSet.of(Lifespan.driverGroup(5)), false))); waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT); assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1))); waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT); TABLE_SCAN_NODE_ID, ImmutableSet.of( newScheduledSplit(3, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 300000, 45), newScheduledSplit(4, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 400000, 54)), ImmutableSet.of(), true))); waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5), Lifespan.driverGroup(7)), ASSERT_WAIT_TIMEOUT);
scan2NodeId, ImmutableSet.of( newScheduledSplit(0, scan2NodeId, Lifespan.driverGroup(3), 0, 1), newScheduledSplit(1, scan2NodeId, Lifespan.driverGroup(3), 100000, 2)), false))); waitUntilEquals(joinOperatorFactoryB::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(buildOperatorFactoryA::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); sqlTaskExecution.addSources(ImmutableList.of(new TaskSource( scan2NodeId, ImmutableSet.of(newScheduledSplit(2, scan2NodeId, Lifespan.driverGroup(3), 200000, 2)), ImmutableSet.of(Lifespan.driverGroup(3)), false))); waitUntilEquals(scanOperatorFactory2::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(buildOperatorFactoryB::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); ImmutableSet.of(newScheduledSplit(3, scan0NodeId, Lifespan.driverGroup(3), 300000, 10)), ImmutableSet.of(Lifespan.driverGroup(3)), false))); waitUntilEquals(scanOperatorFactory0::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(joinOperatorFactoryA::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(joinOperatorFactoryC::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(3)), ASSERT_WAIT_TIMEOUT); ImmutableSet.of(newScheduledSplit(4, scan2NodeId, Lifespan.driverGroup(7), 400000, 2)), ImmutableSet.of(Lifespan.driverGroup(7)),
@Test(timeOut = 30000) public void testRegular() throws Exception { AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime()); TestingTaskResource testingTaskResource = new TestingTaskResource(lastActivityNanos, FailureScenario.NO_FAILURE); HttpRemoteTaskFactory httpRemoteTaskFactory = createHttpRemoteTaskFactory(testingTaskResource); RemoteTask remoteTask = createRemoteTask(httpRemoteTaskFactory); testingTaskResource.setInitialTaskInfo(remoteTask.getTaskInfo()); remoteTask.start(); Lifespan lifespan = Lifespan.driverGroup(3); remoteTask.addSplits(ImmutableMultimap.of(TABLE_SCAN_NODE_ID, new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit(), lifespan))); poll(() -> testingTaskResource.getTaskSource(TABLE_SCAN_NODE_ID) != null); poll(() -> testingTaskResource.getTaskSource(TABLE_SCAN_NODE_ID).getSplits().size() == 1); remoteTask.noMoreSplits(TABLE_SCAN_NODE_ID, lifespan); poll(() -> testingTaskResource.getTaskSource(TABLE_SCAN_NODE_ID).getNoMoreSplitsForLifespan().size() == 1); remoteTask.noMoreSplits(TABLE_SCAN_NODE_ID); poll(() -> testingTaskResource.getTaskSource(TABLE_SCAN_NODE_ID).isNoMoreSplits()); remoteTask.cancel(); poll(() -> remoteTask.getTaskStatus().getState().isDone()); poll(() -> remoteTask.getTaskInfo().getTaskStatus().getState().isDone()); httpRemoteTaskFactory.stop(); }