@Test public void testSimpleQuery() throws Exception { SqlTask sqlTask = createInitialTask(); TaskInfo taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), OptionalInt.empty()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); BufferResult results = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE)).get(); assertEquals(results.isBufferComplete(), false); assertEquals(results.getSerializedPages().size(), 1); assertEquals(results.getSerializedPages().get(0).getPositionCount(), 1); for (boolean moreResults = true; moreResults; moreResults = !results.isBufferComplete()) { results = sqlTask.getTaskResults(OUT, results.getToken() + results.getSerializedPages().size(), new DataSize(1, MEGABYTE)).get(); } assertEquals(results.getSerializedPages().size(), 0); // complete the task by calling abort on it TaskInfo info = sqlTask.abortTaskResults(OUT); assertEquals(info.getOutputBuffers().getState(), BufferState.FINISHED); taskInfo = sqlTask.getTaskInfo(taskInfo.getTaskStatus().getState()).get(1, SECONDS); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); }
private TaskInfo buildTaskInfo() { return new TaskInfo( buildTaskStatus(), initialTaskInfo.getLastHeartbeat(), initialTaskInfo.getOutputBuffers(), initialTaskInfo.getNoMoreSplits(), initialTaskInfo.getStats(), initialTaskInfo.isNeedsPlan()); }
@Test public void testSimpleQuery() throws Exception { try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig())) { TaskId taskId = TASK_ID; TaskInfo taskInfo = createTask(sqlTaskManager, taskId, ImmutableSet.of(SPLIT), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); BufferResult results = sqlTaskManager.getTaskResults(taskId, OUT, 0, new DataSize(1, Unit.MEGABYTE)).get(); assertEquals(results.isBufferComplete(), false); assertEquals(results.getSerializedPages().size(), 1); assertEquals(results.getSerializedPages().get(0).getPositionCount(), 1); for (boolean moreResults = true; moreResults; moreResults = !results.isBufferComplete()) { results = sqlTaskManager.getTaskResults(taskId, OUT, results.getToken() + results.getSerializedPages().size(), new DataSize(1, Unit.MEGABYTE)).get(); } assertEquals(results.isBufferComplete(), true); assertEquals(results.getSerializedPages().size(), 0); // complete the task by calling abort on it TaskInfo info = sqlTaskManager.abortTaskResults(taskId, OUT); assertEquals(info.getOutputBuffers().getState(), BufferState.FINISHED); taskInfo = sqlTaskManager.getTaskInfo(taskId, taskInfo.getTaskStatus().getState()).get(1, TimeUnit.SECONDS); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.FINISHED); } }
processedInputPositions += taskStats.getProcessedInputPositions(); bufferedDataSize += taskInfo.getOutputBuffers().getTotalBufferedBytes(); outputDataSize += taskStats.getOutputDataSize().toBytes(); outputPositions += taskStats.getOutputPositions();
private static boolean allOutputBuffersCreated(StageInfo outputStage) { StageState stageState = outputStage.getState(); // if the stage is already done, then there will be no more buffers if (stageState.isDone()) { return true; } // have all stage tasks been scheduled? if (stageState == StageState.PLANNED || stageState == StageState.SCHEDULING) { return false; } // have all tasks finished adding buffers return outputStage.getTasks().stream() .allMatch(taskInfo -> !taskInfo.getOutputBuffers().getState().canAddBuffers()); }
private synchronized void updateExchangeClient(StageInfo outputStage) { // add any additional output locations if (!outputStage.getState().isDone()) { for (TaskInfo taskInfo : outputStage.getTasks()) { SharedBufferInfo outputBuffers = taskInfo.getOutputBuffers(); List<BufferInfo> buffers = outputBuffers.getBuffers(); if (buffers.isEmpty() || outputBuffers.getState().canAddBuffers()) { // output buffer has not been created yet continue; } Preconditions.checkState(buffers.size() == 1, "Expected a single output buffer for task %s, but found %s", taskInfo.getTaskId(), buffers); TaskId bufferId = Iterables.getOnlyElement(buffers).getBufferId(); URI uri = uriBuilderFrom(taskInfo.getSelf()).appendPath("results").appendPath(bufferId.toString()).build(); exchangeClient.addLocation(uri); } } if (allOutputBuffersCreated(outputStage)) { exchangeClient.noMoreLocations(); } }
@Test public void testBufferCloseOnFinish() throws Exception { SqlTask sqlTask = createInitialTask(); OutputBuffers outputBuffers = INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer(OUT, 0).withNoMoreBufferIds(); updateTask(sqlTask, EMPTY_SOURCES, outputBuffers); CompletableFuture<BufferResult> bufferResult = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE)); assertFalse(bufferResult.isDone()); // close the sources (no splits will ever be added) updateTask(sqlTask, ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.<ScheduledSplit>of(), true)), outputBuffers); // finish the task by calling abort on it TaskInfo taskInfo = sqlTask.abortTaskResults(OUT); // buffer will be closed by cancel event (wait for event to fire) assertTrue(bufferResult.get(1, SECONDS).isBufferComplete()); assertEquals(sqlTask.getTaskInfo().getOutputBuffers().getState(), BufferState.FINISHED); taskInfo = sqlTask.getTaskInfo(taskInfo.getState()).get(1, SECONDS); assertEquals(taskInfo.getState(), TaskState.FINISHED); // verify the buffer is closed bufferResult = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE)); assertTrue(bufferResult.isDone()); assertTrue(bufferResult.get().isBufferComplete()); }
@Test public void testSimpleQuery() throws Exception { SqlTask sqlTask = createInitialTask(); TaskInfo taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getState(), TaskState.RUNNING); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getState(), TaskState.RUNNING); BufferResult results = sqlTask.getTaskResults(OUT, 0, new DataSize(1, MEGABYTE)).get(); assertEquals(results.isBufferComplete(), false); assertEquals(results.getPages().size(), 1); assertEquals(results.getPages().get(0).getPositionCount(), 1); results = sqlTask.getTaskResults(OUT, results.getToken() + results.getPages().size(), new DataSize(1, MEGABYTE)).get(); assertEquals(results.isBufferComplete(), true); assertEquals(results.getPages().size(), 0); // complete the task by calling abort on it TaskInfo info = sqlTask.abortTaskResults(OUT); assertEquals(info.getOutputBuffers().getState(), BufferState.FINISHED); taskInfo = sqlTask.getTaskInfo(taskInfo.getState()).get(1, SECONDS); assertEquals(taskInfo.getState(), TaskState.FINISHED); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getState(), TaskState.FINISHED); }
for (TaskInfo task : stage.getTasks()) { int bufferedPages = 0; for (BufferInfo bufferInfo : task.getOutputBuffers().getBuffers()) { bufferedPages += bufferInfo.getBufferedPages();
@Test public void testSimpleQuery() throws Exception { try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig())) { TaskId taskId = TASK_ID; TaskInfo taskInfo = sqlTaskManager.updateTask(TEST_SESSION, taskId, Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)), INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getState(), TaskState.RUNNING); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getState(), TaskState.RUNNING); BufferResult results = sqlTaskManager.getTaskResults(taskId, OUT, 0, new DataSize(1, Unit.MEGABYTE)).get(); assertEquals(results.isBufferComplete(), false); assertEquals(results.getPages().size(), 1); assertEquals(results.getPages().get(0).getPositionCount(), 1); results = sqlTaskManager.getTaskResults(taskId, OUT, results.getToken() + results.getPages().size(), new DataSize(1, Unit.MEGABYTE)).get(); assertEquals(results.isBufferComplete(), true); assertEquals(results.getPages().size(), 0); // complete the task by calling abort on it TaskInfo info = sqlTaskManager.abortTaskResults(taskId, OUT); assertEquals(info.getOutputBuffers().getState(), SharedBuffer.BufferState.FINISHED); taskInfo = sqlTaskManager.getTaskInfo(taskId, taskInfo.getState()).get(1, TimeUnit.SECONDS); assertEquals(taskInfo.getState(), TaskState.FINISHED); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getState(), TaskState.FINISHED); } }
/** * Move the task directly to the failed state */ private void failTask(Throwable cause) { TaskInfo taskInfo = getTaskInfo(); if (!taskInfo.getState().isDone()) { log.debug(cause, "Remote task failed: %s", taskInfo.getSelf()); } updateTaskInfo(new TaskInfo(taskInfo.getTaskId(), taskInfo.getTaskInstanceId(), TaskInfo.MAX_VERSION, TaskState.FAILED, taskInfo.getSelf(), taskInfo.getLastHeartbeat(), taskInfo.getOutputBuffers(), taskInfo.getNoMoreSplits(), taskInfo.getStats(), ImmutableList.of(toFailure(cause)), taskInfo.isNeedsPlan())); }
uri, taskInfo.getLastHeartbeat(), taskInfo.getOutputBuffers(), taskInfo.getNoMoreSplits(), taskInfo.getStats(),