@SuppressWarnings("unchecked") public T newPlugin(Injector injector) { return (T) new FileInputRunner((FileInputPlugin) injector.getInstance(impl)); } };
public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) { if (taskCount == 0) { throw new NoSampleException("No input files to guess"); } input.run(inputTaskSource, null, 0, new PageOutput() { @Override public void add(Page page) { throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration"); // TODO exception class } @Override public void finish() {} @Override public void close() {} }); throw new AssertionError("Guess executor must throw GuessedNoticeError"); } });
@Override public ConfigDiff guess(ConfigSource config) { return guess(Exec.newConfigSource(), config); }
public RunnerControl(RunnerTask task, InputPlugin.Control nextControl) { this.task = task; // create plugins earlier than run() to throw exceptions early this.decoderPlugins = newDecoderPlugins(task); this.parserPlugin = newParserPlugin(task); this.nextControl = nextControl; }
final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample)); ConfigDiff guessed; try { input.transaction(guessInputConfig, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) { if (taskCount == 0) {
runner.transaction(samplingInputConfig, new InputPlugin.Control() { public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount) { if (taskCount == 0) {
inputTaskSource = FileInputRunner.getFileInputTaskSource(resume.getInputTaskSource()); } else { inputTaskSource = resume.getInputTaskSource();
MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin( new LinkedList<Buffer>(Arrays.asList(buffers))); final FileInputRunner runner = new FileInputRunner(fileInputPlugin); runner.transaction(config, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
private void doTest(ConfigSource config) { ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); assertRecords(config, output); }
@Override public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) { final RunnerTask task = taskSource.loadTask(RunnerTask.class); List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task); ParserPlugin parserPlugin = newParserPlugin(task); final TransactionalFileInput tran = PluginWrappers.transactionalFileInput( fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex)); try (CloseResource closer = new CloseResource(tran)) { try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) { FileInput fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), tran); closer.closeThis(fileInput); parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output); TaskReport report = tran.commit(); // TODO check output.finish() is called. wrap aborter.dontAbort(); return report; } } }
MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin( new LinkedList<Buffer>(Arrays.asList(buffers))); final FileInputRunner runner = new FileInputRunner(fileInputPlugin); runner.transaction(config, new InputPlugin.Control() { public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
@Test public void usePathAsHighPriorityThanPathPrefix() { ConfigSource config = this.config.deepCopy() .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX)) .set("path_prefix", "foo"); // path_prefix has the bad value, if path_prefix is chosen, expected result will be failed ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path")); assertRecords(config, output); }
public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) { List<TaskReport> reports = new ArrayList<>(); reports.add(runner.run(inputTaskSource, schema, 0, output)); return reports; } });
ex); pluginMainObject = new FileInputRunner(fileInputPluginMainObject); } else if (FileOutputPlugin.class.isAssignableFrom(pluginMainClass)) { final FileOutputPlugin fileOutputPluginMainObject;
@Test public void usePath() { ConfigSource config = this.config.deepCopy() .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX)) .set("path_prefix", null); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path")); assertRecords(config, output); }
public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) { List<TaskReport> reports = new ArrayList<>(); reports.add(runner.run(inputTaskSource, schema, 0, output)); return reports; } });
private PreviewResult doPreview(ConfigSource config) { PreviewTask task = config.loadConfig(PreviewTask.class); InputPlugin inputPlugin = newInputPlugin(task); List<FilterPlugin> filterPlugins = newFilterPlugins(task); if (inputPlugin instanceof FileInputRunner) { // file input runner Buffer sample = SamplingParserPlugin.runFileInputSampling( (FileInputRunner) inputPlugin, config.getNested("in"), createSampleBufferConfigFromExecConfig(task.getExecConfig())); FileInputRunner previewRunner = new FileInputRunner(new BufferFileInputPlugin(sample)); return doPreview(task, previewRunner, filterPlugins); } else { return doPreview(task, inputPlugin, filterPlugins); } }
@Test public void useTotalFileCountLimit() throws Exception { ConfigSource config = this.config.deepCopy().set("total_file_count_limit", 0); ConfigDiff configDiff = runner.transaction(config, new Control(runner, output)); assertNull(configDiff.get(String.class, "last_path")); assertEquals(0, getRecords(config, output).size()); }
@Before public void createResources() { config = runtime.getExec().newConfigSource() .set("type", "s3") .set("bucket", EMBULK_S3_TEST_BUCKET) .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX) .set("parser", parserConfig(schemaConfig())); runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class)); output = new TestPageBuilderReader.MockPageOutput(); }