private boolean isAllScatterOutputFinished(ProcessState state, int taskIndex) { for (int i = 0; i < scatterCount; i++) { int outputTaskIndex = taskIndex * scatterCount + i; if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) { return false; } } return true; }
private void showProgress(ProcessState state, int taskCount) { int started = 0; int finished = 0; for (int i = 0; i < taskCount; i++) { if (state.getOutputTaskState(i).isStarted()) { started++; } if (state.getOutputTaskState(i).isFinished()) { finished++; } } logger.info(String.format("{done:%3d / %d, running: %d}", finished, taskCount, started - finished)); }
@Override protected Future<Throwable> startInputTask(final ProcessTask task, final ProcessState state, final int taskIndex) { if (state.getOutputTaskState(taskIndex).isCommitted()) { logger.warn("Skipped resumed task {}", taskIndex); return null; // resumed } return executor.submit(new Callable<Throwable>() { public Throwable call() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) { Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() { public void started() { state.getInputTaskState(taskIndex).start(); state.getOutputTaskState(taskIndex).start(); } public void inputCommitted(TaskReport report) { state.getInputTaskState(taskIndex).setTaskReport(report); } public void outputCommitted(TaskReport report) { state.getOutputTaskState(taskIndex).setTaskReport(report); } }); return null; } finally { state.getInputTaskState(taskIndex).finish(); state.getOutputTaskState(taskIndex).finish(); } } }); } }
public void openOutputs(OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource) { for (int i = 0; i < scatterCount; i++) { int outputTaskIndex = taskIndex * scatterCount + i; if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) { TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput( outputPlugin.open(outputTaskSource, outputSchema, outputTaskIndex)); trans[i] = tran; closeThese[i].closeThis(tran); } } }
public TaskReport commit() { completeWorkers(); for (int i = 0; i < scatterCount; i++) { if (trans[i] != null) { int outputTaskIndex = taskIndex * scatterCount + i; TaskReport outputTaskReport = trans[i].commit(); trans[i] = null; // don't abort if (outputTaskReport == null) { outputTaskReport = Exec.newTaskReport(); } state.getOutputTaskState(outputTaskIndex).setTaskReport(outputTaskReport); } } return null; }
public Throwable call() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) { Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() { public void started() { state.getInputTaskState(taskIndex).start(); state.getOutputTaskState(taskIndex).start(); } public void inputCommitted(TaskReport report) { state.getInputTaskState(taskIndex).setTaskReport(report); } public void outputCommitted(TaskReport report) { state.getOutputTaskState(taskIndex).setTaskReport(report); } }); return null; } finally { state.getInputTaskState(taskIndex).finish(); state.getOutputTaskState(taskIndex).finish(); } } });
private void runInputTask(ExecSession exec, ProcessTask task, ProcessState state, int taskIndex) { InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType()); List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes()); OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType()); try (ScatterTransactionalPageOutput tran = new ScatterTransactionalPageOutput(state, taskIndex, scatterCount)) { tran.openOutputs(outputPlugin, task.getOutputSchema(), task.getOutputTaskSource()); try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { tran.openFilters(filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources()); tran.startWorkers(outputExecutor); // started state.getInputTaskState(taskIndex).start(); for (int i = 0; i < scatterCount; i++) { state.getOutputTaskState(taskIndex * scatterCount + i).start(); } TaskReport inputTaskReport = inputPlugin.run(task.getInputTaskSource(), task.getInputSchema(), taskIndex, tran); // inputCommitted if (inputTaskReport == null) { inputTaskReport = exec.newTaskReport(); } state.getInputTaskState(taskIndex).setTaskReport(inputTaskReport); // outputCommitted tran.commit(); aborter.dontAbort(); } } finally { state.getInputTaskState(taskIndex).finish(); state.getOutputTaskState(taskIndex).finish(); } } }