public int countUncommittedInputTasks() { if (inputTaskStates == null) { // not initialized return 0; } int count = 0; for (TaskState inputTaskState : inputTaskStates) { if (!inputTaskState.isCommitted()) { count++; } } return count; }
public int countUncommittedOutputTasks() { if (outputTaskStates == null) { // not initialized return 0; } int count = 0; for (TaskState outputTaskState : outputTaskStates) { if (!outputTaskState.isCommitted()) { count++; } } return count; }
public boolean isAllTasksCommitted() { // here can't assume that input tasks are committed when output tasks are // committed because that's controlled by executor plugins. some executor // plugins (especially mapreduce executor) may commit output tasks even // when some input tasks failed. This is asemantically allowed behavior for // executor plugins (as long as output plugin is atomic and idempotent). if (inputTaskStates == null || outputTaskStates == null) { // not initialized return false; } for (TaskState inputTaskState : inputTaskStates) { if (!inputTaskState.isCommitted()) { return false; } } for (TaskState outputTaskState : outputTaskStates) { if (!outputTaskState.isCommitted()) { return false; } } return true; }
private boolean isAllScatterOutputFinished(ProcessState state, int taskIndex) { for (int i = 0; i < scatterCount; i++) { int outputTaskIndex = taskIndex * scatterCount + i; if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) { return false; } } return true; }
@Override protected Future<Throwable> startInputTask(final ProcessTask task, final ProcessState state, final int taskIndex) { if (state.getOutputTaskState(taskIndex).isCommitted()) { logger.warn("Skipped resumed task {}", taskIndex); return null; // resumed } return executor.submit(new Callable<Throwable>() { public Throwable call() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) { Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() { public void started() { state.getInputTaskState(taskIndex).start(); state.getOutputTaskState(taskIndex).start(); } public void inputCommitted(TaskReport report) { state.getInputTaskState(taskIndex).setTaskReport(report); } public void outputCommitted(TaskReport report) { state.getOutputTaskState(taskIndex).setTaskReport(report); } }); return null; } finally { state.getInputTaskState(taskIndex).finish(); state.getOutputTaskState(taskIndex).finish(); } } }); } }
public void openOutputs(OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource) { for (int i = 0; i < scatterCount; i++) { int outputTaskIndex = taskIndex * scatterCount + i; if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) { TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput( outputPlugin.open(outputTaskSource, outputSchema, outputTaskIndex)); trans[i] = tran; closeThese[i].closeThis(tran); } } }