@Override public CompletableFuture<AccumulatorReport> retrievePayload(ResourceID resourceID) { validateRunsInMainThread(); JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID); if (jobManagerConnection != null) { JobID jobId = jobManagerConnection.getJobID(); List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16); Iterator<Task> allTasks = taskSlotTable.getTasks(jobId); while (allTasks.hasNext()) { Task task = allTasks.next(); accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot()); } return CompletableFuture.completedFuture(new AccumulatorReport(accumulatorSnapshots)); } else { return CompletableFuture.completedFuture(new AccumulatorReport(Collections.emptyList())); } } }
@Override public CompletableFuture<AccumulatorReport> retrievePayload(ResourceID resourceID) { validateRunsInMainThread(); JobManagerConnection jobManagerConnection = jobManagerConnections.get(resourceID); if (jobManagerConnection != null) { JobID jobId = jobManagerConnection.getJobID(); List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16); Iterator<Task> allTasks = taskSlotTable.getTasks(jobId); while (allTasks.hasNext()) { Task task = allTasks.next(); accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot()); } return CompletableFuture.completedFuture(new AccumulatorReport(accumulatorSnapshots)); } else { return CompletableFuture.completedFuture(new AccumulatorReport(Collections.emptyList())); } } }
@Override public CompletableFuture<AccumulatorReport> retrievePayload(ResourceID resourceID) { validateRunsInMainThread(); JobManagerConnection jobManagerConnection = jobManagerTable.get(resourceID); if (jobManagerConnection != null) { JobID jobId = jobManagerConnection.getJobID(); List<AccumulatorSnapshot> accumulatorSnapshots = new ArrayList<>(16); Iterator<Task> allTasks = taskSlotTable.getTasks(jobId); while (allTasks.hasNext()) { Task task = allTasks.next(); accumulatorSnapshots.add(task.getAccumulatorRegistry().getSnapshot()); } return CompletableFuture.completedFuture(new AccumulatorReport(accumulatorSnapshots)); } else { return CompletableFuture.completedFuture(new AccumulatorReport(Collections.emptyList())); } } }
private void unregisterTaskAndNotifyFinalState( final JobMasterGateway jobMasterGateway, final ExecutionAttemptID executionAttemptID) { Task task = taskSlotTable.removeTask(executionAttemptID); if (task != null) { if (!task.getExecutionState().isTerminal()) { try { task.failExternally(new IllegalStateException("Task is being remove from TaskManager.")); } catch (Exception e) { log.error("Could not properly fail task.", e); } } log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.", task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId()); AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot(); updateTaskExecutionState( jobMasterGateway, new TaskExecutionState( task.getJobID(), task.getExecutionId(), task.getExecutionState(), task.getFailureCause(), accumulatorSnapshot, task.getMetricGroup().getIOMetricGroup().createSnapshot())); } else { log.error("Cannot find task with ID {} to unregister.", executionAttemptID); } }
private void unregisterTaskAndNotifyFinalState( final JobMasterGateway jobMasterGateway, final ExecutionAttemptID executionAttemptID) { Task task = taskSlotTable.removeTask(executionAttemptID); if (task != null) { if (!task.getExecutionState().isTerminal()) { try { task.failExternally(new IllegalStateException("Task is being remove from TaskManager.")); } catch (Exception e) { log.error("Could not properly fail task.", e); } } log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.", task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId()); AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot(); updateTaskExecutionState( jobMasterGateway, new TaskExecutionState( task.getJobID(), task.getExecutionId(), task.getExecutionState(), task.getFailureCause(), accumulatorSnapshot, task.getMetricGroup().getIOMetricGroup().createSnapshot())); } else { log.error("Cannot find task with ID {} to unregister.", executionAttemptID); } }
private void unregisterTaskAndNotifyFinalState( final UUID jobMasterLeaderId, final JobMasterGateway jobMasterGateway, final ExecutionAttemptID executionAttemptID) { Task task = taskSlotTable.removeTask(executionAttemptID); if (task != null) { if (!task.getExecutionState().isTerminal()) { try { task.failExternally(new IllegalStateException("Task is being remove from TaskManager.")); } catch (Exception e) { log.error("Could not properly fail task.", e); } } log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.", task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId()); AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot(); updateTaskExecutionState( jobMasterLeaderId, jobMasterGateway, new TaskExecutionState( task.getJobID(), task.getExecutionId(), task.getExecutionState(), task.getFailureCause(), accumulatorSnapshot, task.getMetricGroup().getIOMetricGroup().createSnapshot())); } else { log.error("Cannot find task with ID {} to unregister.", executionAttemptID); } }
task.getTaskInfo().getIndexOfThisSubtask()); AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();