public void removeOldTasks() { DateTime oldestAllowedTask = DateTime.now().minus(infoCacheTime.toMillis()); for (TaskInfo taskInfo : filter(transform(tasks.asMap().values(), SqlTask::getTaskInfo), notNull())) { TaskId taskId = taskInfo.getTaskStatus().getTaskId(); try { DateTime endTime = taskInfo.getStats().getEndTime(); if (endTime != null && endTime.isBefore(oldestAllowedTask)) { tasks.asMap().remove(taskId); } } catch (RuntimeException e) { log.warn(e, "Error while inspecting age of complete task %s", taskId); } } }
public synchronized Duration getTotalCpuTime() { long millis = getAllTasks().stream() .mapToLong(task -> task.getTaskInfo().getStats().getTotalCpuTime().toMillis()) .sum(); return new Duration(millis, TimeUnit.MILLISECONDS); }
TaskStats taskStats = taskInfo.getStats();
StageStats stageStats = stageInfo.get().getStageStats(); double avgPositionsPerTask = stageInfo.get().getTasks().stream().mapToLong(task -> task.getStats().getProcessedInputPositions()).average().orElse(Double.NaN); double squaredDifferences = stageInfo.get().getTasks().stream().mapToDouble(task -> Math.pow(task.getStats().getProcessedInputPositions() - avgPositionsPerTask, 2)).sum(); double sdAmongTasks = Math.sqrt(squaredDifferences / stageInfo.get().getTasks().size());
private TaskStats getTaskStats(TaskHolder taskHolder) { TaskInfo finalTaskInfo = taskHolder.getFinalTaskInfo(); if (finalTaskInfo != null) { return finalTaskInfo.getStats(); } SqlTaskExecution taskExecution = taskHolder.getTaskExecution(); if (taskExecution != null) { return taskExecution.getTaskContext().getTaskStats(); } // if the task completed without creation, set end time DateTime endTime = taskStateMachine.getState().isDone() ? DateTime.now() : null; return new TaskStats(taskStateMachine.getCreatedTime(), endTime); }
@Test public void testCancel() { SqlTask sqlTask = createInitialTask(); TaskInfo taskInfo = sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(), createInitialEmptyOutputBuffers(PARTITIONED) .withBuffer(OUT, 0) .withNoMoreBufferIds(), OptionalInt.empty()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTask.cancel(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); assertNotNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTask.getTaskInfo(); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); assertNotNull(taskInfo.getStats().getEndTime()); }
@Test public void testCancel() { try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig())) { TaskId taskId = TASK_ID; TaskInfo taskInfo = createTask(sqlTaskManager, taskId, createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.cancelTask(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); assertNotNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.CANCELED); assertNotNull(taskInfo.getStats().getEndTime()); } }
@Test public void testAbort() { try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig())) { TaskId taskId = TASK_ID; TaskInfo taskInfo = createTask(sqlTaskManager, taskId, createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds()); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.RUNNING); assertNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.abortTask(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.ABORTED); assertNotNull(taskInfo.getStats().getEndTime()); taskInfo = sqlTaskManager.getTaskInfo(taskId); assertEquals(taskInfo.getTaskStatus().getState(), TaskState.ABORTED); assertNotNull(taskInfo.getStats().getEndTime()); } }
private TaskInfo buildTaskInfo() { return new TaskInfo( buildTaskStatus(), initialTaskInfo.getLastHeartbeat(), initialTaskInfo.getOutputBuffers(), initialTaskInfo.getNoMoreSplits(), initialTaskInfo.getStats(), initialTaskInfo.isNeedsPlan()); }
TaskStats taskStats = taskInfo.getStats();
TaskStats taskStats = taskInfo.getStats();
private static StageCpuDistribution computeCpuDistribution(StageInfo stageInfo) { Distribution cpuDistribution = new Distribution(); for (TaskInfo taskInfo : stageInfo.getTasks()) { cpuDistribution.add(taskInfo.getStats().getTotalCpuTime().toMillis()); } DistributionSnapshot snapshot = cpuDistribution.snapshot(); return new StageCpuDistribution( stageInfo.getStageId().getId(), stageInfo.getTasks().size(), snapshot.getP25(), snapshot.getP50(), snapshot.getP75(), snapshot.getP90(), snapshot.getP95(), snapshot.getP99(), snapshot.getMin(), snapshot.getMax(), (long) snapshot.getTotal(), snapshot.getTotal() / snapshot.getCount()); } }
Duration fullGcTime = new Duration(0, MILLISECONDS); if (taskHolder.getFinalTaskInfo() != null) { TaskStats taskStats = taskHolder.getFinalTaskInfo().getStats(); queuedPartitionedDrivers = taskStats.getQueuedPartitionedDrivers(); runningPartitionedDrivers = taskStats.getRunningPartitionedDrivers();
TaskStats stats = taskInfo.getStats(); TaskStatus taskStatus = taskInfo.getTaskStatus(); table.addRow(
public synchronized long getMemoryReservation() { return getAllTasks().stream() .mapToLong(task -> task.getTaskInfo().getStats().getMemoryReservation().toBytes()) .sum(); }
private synchronized void updateMemoryUsage(TaskInfo taskInfo) { long currentMemory = taskInfo.getStats().getMemoryReservation().toBytes(); long deltaMemoryInBytes = currentMemory - previousMemory; previousMemory = currentMemory; stateMachine.updateMemoryUsage(deltaMemoryInBytes); } }
@Override public synchronized void stateChanged(TaskInfo newState) { long currentMemory = newState.getStats().getMemoryReservation().toBytes(); long deltaMemoryInBytes = currentMemory - previousMemory; previousMemory = currentMemory; stateMachine.updateMemoryUsage(deltaMemoryInBytes); } }
private static boolean allQueriesBlocked(List<QueryInfo> current) { boolean allDriversBlocked = current.stream() .flatMap(query -> getAllStages(query.getOutputStage()).stream()) .flatMap(stage -> stage.getTasks().stream()) .flatMap(task -> task.getStats().getPipelines().stream()) .flatMap(pipeline -> pipeline.getDrivers().stream()) .allMatch(DriverStats::isFullyBlocked); boolean waitingForMemory = current.stream().allMatch(TestMemoryManager::atLeastOneOperatorWaitingForMemory); return allDriversBlocked && waitingForMemory; }
@Override public int getQueuedPartitionedSplitCount() { TaskInfo taskInfo = this.taskInfo.get(); if (taskInfo.getState().isDone()) { return 0; } return getPendingSourceSplitCount() + taskInfo.getStats().getQueuedPartitionedDrivers(); }