public ConfigSource getSessionExecConfig() { return newConfigSource() .set("transaction_time", transactionTime); }
public void cleanup(final ConfigSource config, final ResumeState resume) { try { ExecSession exec = ExecSession.builder(injector).fromExecConfig(resume.getExecSessionConfigSource()).build(); Exec.doWith(exec, new ExecAction<Void>() { public Void run() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName("cleanup")) { doCleanup(config, resume); return null; } } }); exec.cleanup(); } catch (ExecutionException ex) { if (ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); } if (ex.getCause() instanceof Error) { throw (Error) ex.getCause(); } throw new RuntimeException(ex.getCause()); } }
private static List<TaskReport> buildReports(List<Optional<TaskReport>> optionalReports, ExecSession session) { ImmutableList.Builder<TaskReport> reports = ImmutableList.builder(); for (Optional<TaskReport> report : optionalReports) { reports.add(report.or(session.newTaskReport())); } return reports.build(); }
public PreviewResult preview(ConfigSource config) { this.logger.info("Started Embulk v" + EmbulkVersion.VERSION); ExecSession exec = newExecSession(config); try { return previewExecutor.preview(exec, config); } finally { exec.cleanup(); } }
private void runInputTask(ExecSession exec, ProcessTask task, ProcessState state, int taskIndex) { InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType()); List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes()); OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType()); try (ScatterTransactionalPageOutput tran = new ScatterTransactionalPageOutput(state, taskIndex, scatterCount)) { tran.openOutputs(outputPlugin, task.getOutputSchema(), task.getOutputTaskSource()); try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { tran.openFilters(filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources()); tran.startWorkers(outputExecutor); // started state.getInputTaskState(taskIndex).start(); for (int i = 0; i < scatterCount; i++) { state.getOutputTaskState(taskIndex * scatterCount + i).start(); } TaskReport inputTaskReport = inputPlugin.run(task.getInputTaskSource(), task.getInputSchema(), taskIndex, tran); // inputCommitted if (inputTaskReport == null) { inputTaskReport = exec.newTaskReport(); } state.getInputTaskState(taskIndex).setTaskReport(inputTaskReport); // outputCommitted tran.commit(); aborter.dontAbort(); } } finally { state.getInputTaskState(taskIndex).finish(); state.getOutputTaskState(taskIndex).finish(); } } }
private ExecSession newExecSession(ConfigSource config) { ConfigSource execConfig = config.deepCopy().getNestedOrGetEmpty("exec"); return ExecSession.builder(injector).fromExecConfig(execConfig).build(); }
public ExecSession forPreview() { return new ExecSession(this, true); }
public PreviewResult preview(ExecSession exec, final ConfigSource config) { try { return Exec.doWith(exec.forPreview(), new ExecAction<PreviewResult>() { public PreviewResult run() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName("preview")) { return doPreview(config); } } }); } catch (Exception ex) { if (ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); } throw new RuntimeException(ex.getCause()); } }
public ConfigDiff guess(ConfigSource config) { this.logger.info("Started Embulk v" + EmbulkVersion.VERSION); ExecSession exec = newExecSession(config); try { return guessExecutor.guess(exec, config); } finally { exec.cleanup(); } }
public EmbulkTestRuntime() { super(new TestRuntimeModule()); Injector injector = getInjector(); ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class)); this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build(); }
public ExecSession build() { if (transactionTime == null) { transactionTime = Timestamp.ofEpochMilli(System.currentTimeMillis()); // TODO get nanoseconds for default } return new ExecSession(injector, transactionTime, Optional.ofNullable(loggerFactory)); } }
public static ConfigSource newConfigSource() { return session().newConfigSource(); }
public ExecutionResult resume(final ConfigSource config, final ResumeState resume) { try { ExecSession exec = ExecSession.builder(injector).fromExecConfig(resume.getExecSessionConfigSource()).build(); ExecutionResult result = Exec.doWith(exec, new ExecAction<ExecutionResult>() { public ExecutionResult run() { try (SetCurrentThreadName dontCare = new SetCurrentThreadName("resume")) { return doResume(config, resume); } } }); exec.cleanup(); return result; } catch (ExecutionException ex) { if (ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); } if (ex.getCause() instanceof Error) { throw (Error) ex.getCause(); } throw new RuntimeException(ex.getCause()); } }
public ResumableResult runResumable(ConfigSource config) { this.logger.info("Started Embulk v" + EmbulkVersion.VERSION); ExecSession exec = newExecSession(config); try { ExecutionResult result; try { result = bulkLoader.run(exec, config); } catch (PartialExecutionException partial) { return new ResumableResult(partial); } return new ResumableResult(result); } finally { try { exec.cleanup(); } catch (Exception ex) { // TODO add this exception to ExecutionResult.getIgnoredExceptions // or partial.addSuppressed ex.printStackTrace(System.err); } } }
public static TaskReport newTaskReport() { return session().newTaskReport(); }
private ConfigSource config() { return runtime.getExec().newConfigSource(); }
public void evaluate() throws Throwable { try { Exec.doWith(exec, new ExecAction<Void>() { public Void run() { try { superStatement.evaluate(); } catch (Throwable ex) { throw new RuntimeExecutionException(ex); } return null; } }); } catch (RuntimeException ex) { throw ex.getCause(); } finally { exec.cleanup(); } } };
public static void process(ExecSession exec, int taskIndex, InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource, List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources, OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource, ProcessStateCallback callback) { TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput( outputPlugin.open(outputTaskSource, outputSchema, taskIndex)); callback.started(); // here needs to use try-with-resource to add exception happend at close() or abort() // to suppressed exception. otherwise exception happend at close() or abort() overwrites // essential exception. try (CloseResource closer = new CloseResource(tran)) { try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran); closer.closeThis(filtered); TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered); if (inputTaskReport == null) { inputTaskReport = exec.newTaskReport(); } callback.inputCommitted(inputTaskReport); TaskReport outputTaskReport = tran.commit(); aborter.dontAbort(); if (outputTaskReport == null) { outputTaskReport = exec.newTaskReport(); } callback.outputCommitted(outputTaskReport); // TODO check output.finish() is called. wrap or abstract } } }
private ConfigSource config() { return runtime.getExec().newConfigSource(); }
public ExecutionResult run(ConfigSource config) { this.logger.info("Started Embulk v" + EmbulkVersion.VERSION); ExecSession exec = newExecSession(config); try { return bulkLoader.run(exec, config); } catch (PartialExecutionException partial) { try { bulkLoader.cleanup(config, partial.getResumeState()); } catch (Throwable ex) { partial.addSuppressed(ex); } throw partial; } finally { try { exec.cleanup(); } catch (Exception ex) { // TODO add this exception to ExecutionResult.getIgnoredExceptions // or partial.addSuppressed ex.printStackTrace(System.err); } } }