@JsonCreator public static TaskId valueOf(String taskId) { return new TaskId(taskId); }
@JsonCreator public static TaskId valueOf(String taskId) { return new TaskId(taskId); }
private Builder(Executor notificationExecutor, ScheduledExecutorService yieldExecutor, Session session) { this.notificationExecutor = notificationExecutor; this.yieldExecutor = yieldExecutor; this.session = session; this.taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor); }
private Builder(Executor notificationExecutor, ScheduledExecutorService yieldExecutor, Session session) { this.notificationExecutor = notificationExecutor; this.yieldExecutor = yieldExecutor; this.session = session; this.taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), notificationExecutor); }
public static TaskContext createTaskContext(DefaultQueryContext queryContext, Executor executor, Session session) { return createTaskContext(queryContext, session, new TaskStateMachine(new TaskId("query", 0, 0), executor)); }
public static TaskContext createTaskContext(QueryContext queryContext, Executor executor, Session session) { return createTaskContext(queryContext, session, new TaskStateMachine(new TaskId("query", 0, 0), executor)); }
public synchronized Optional<RemoteTask> scheduleTask(Node node, int partition, OptionalInt totalPartitions) { requireNonNull(node, "node is null"); if (stateMachine.getState().isDone()) { return Optional.empty(); } checkState(!splitsScheduled.get(), "scheduleTask can not be called once splits have been scheduled"); return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageId(), partition), ImmutableMultimap.of(), totalPartitions)); }
public synchronized Optional<RemoteTask> scheduleTask(Node node, int partition, OptionalInt totalPartitions) { requireNonNull(node, "node is null"); if (stateMachine.getState().isDone()) { return Optional.empty(); } checkState(!splitsScheduled.get(), "scheduleTask can not be called once splits have been scheduled"); return Optional.of(scheduleTask(node, new TaskId(stateMachine.getStageId(), partition), ImmutableMultimap.of(), totalPartitions)); }
private void createTask(TaskSpecification specification) { if (specification.getType() == LEAF) { runningTasks.put(specification, new LeafTask( taskExecutor, specification, new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } else { runningTasks.put(specification, new IntermediateTask( taskExecutor, specification, new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } }
private void createTask(TaskSpecification specification) { if (specification.getType() == LEAF) { runningTasks.put(specification, new LeafTask( taskExecutor, specification, new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } else { runningTasks.put(specification, new IntermediateTask( taskExecutor, specification, new TaskId(specification.getName(), 0, runningTasks.get(specification).size() + completedTasks.get(specification).size()))); } }
@Test public void testUpdateLevelWithCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); long quantaNanos = MINUTES.toNanos(10); handle0.addScheduledNanos(quantaNanos); long cappedNanos = Math.min(quantaNanos, LEVEL_CONTRIBUTION_CAP); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long thisLevelTime = Math.min(SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]), cappedNanos); assertEquals(splitQueue.getLevelScheduledTime(i), thisLevelTime); cappedNanos -= thisLevelTime; } }
@Test public void testUpdateLevelWithCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); long quantaNanos = MINUTES.toNanos(10); handle0.addScheduledNanos(quantaNanos); long cappedNanos = Math.min(quantaNanos, LEVEL_CONTRIBUTION_CAP); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long thisLevelTime = Math.min(SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]), cappedNanos); assertEquals(splitQueue.getLevelScheduledTime(i), thisLevelTime); cappedNanos -= thisLevelTime; } }
@Test public void testSingleNode() { FixedCountScheduler nodeScheduler = new FixedCountScheduler( (node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask( new TaskId("test", 1, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {}))), generateRandomNodes(1)); ScheduleResult result = nodeScheduler.schedule(); assertTrue(result.isFinished()); assertTrue(result.getBlocked().isDone()); assertEquals(result.getNewTasks().size(), 1); assertTrue(result.getNewTasks().iterator().next().getNodeId().equals("other 0")); }
@Test public void testMultipleNodes() { FixedCountScheduler nodeScheduler = new FixedCountScheduler( (node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask( new TaskId("test", 1, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {}))), generateRandomNodes(5)); ScheduleResult result = nodeScheduler.schedule(); assertTrue(result.isFinished()); assertTrue(result.getBlocked().isDone()); assertEquals(result.getNewTasks().size(), 5); assertEquals(result.getNewTasks().stream().map(RemoteTask::getNodeId).collect(toImmutableSet()).size(), 5); }
@Test public void testSingleNode() { FixedCountScheduler nodeScheduler = new FixedCountScheduler( (node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask( new TaskId("test", 1, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {}))), generateRandomNodes(1)); ScheduleResult result = nodeScheduler.schedule(); assertTrue(result.isFinished()); assertTrue(result.getBlocked().isDone()); assertEquals(result.getNewTasks().size(), 1); assertTrue(result.getNewTasks().iterator().next().getNodeId().equals("other 0")); }
@Test public void testLevelContributionCap() { MultilevelSplitQueue splitQueue = new MultilevelSplitQueue(2); TaskHandle handle0 = new TaskHandle(new TaskId("test0", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); TaskHandle handle1 = new TaskHandle(new TaskId("test1", 0, 0), splitQueue, () -> 1, 1, new Duration(1, SECONDS), OptionalInt.empty()); for (int i = 0; i < (LEVEL_THRESHOLD_SECONDS.length - 1); i++) { long levelAdvanceTime = SECONDS.toNanos(LEVEL_THRESHOLD_SECONDS[i + 1] - LEVEL_THRESHOLD_SECONDS[i]); handle0.addScheduledNanos(levelAdvanceTime); assertEquals(handle0.getPriority().getLevel(), i + 1); handle1.addScheduledNanos(levelAdvanceTime); assertEquals(handle1.getPriority().getLevel(), i + 1); assertEquals(splitQueue.getLevelScheduledTime(i), 2 * Math.min(levelAdvanceTime, LEVEL_CONTRIBUTION_CAP)); assertEquals(splitQueue.getLevelScheduledTime(i + 1), 0); } }
private OperatorContext createContexts(SqlTask sqlTask) { TaskContext taskContext = sqlTask.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q", 1, 1), executor), session, false, false, OptionalInt.empty()); PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false); DriverContext driverContext = pipelineContext.addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na"); return operatorContext; }
private OperatorContext createContexts(SqlTask sqlTask) { TaskContext taskContext = sqlTask.getQueryContext().addTaskContext(new TaskStateMachine(new TaskId("q", 1, 1), executor), session, false, false, OptionalInt.empty()); PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false); DriverContext driverContext = pipelineContext.addDriverContext(); OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("na"), "na"); return operatorContext; }
private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) { return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {}), true); }
private RemoteTask createRemoteTask(HttpRemoteTaskFactory httpRemoteTaskFactory) { return httpRemoteTaskFactory.createRemoteTask( TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://fake.invalid/"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), OptionalInt.empty(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {}), true); }