public SingleFileProvider(PluginTask task, int taskIndex) { this.client = newS3Client(task); this.bucket = task.getBucket(); this.iterator = task.getFiles().get(taskIndex).iterator(); this.retryExec = retryExecutorFrom(task); }
public FileList build() { try { stream.close(); } catch (IOException ex) { throw Throwables.propagate(ex); } return new FileList(binary.toByteArray(), getSplits(entries), Optional.ofNullable(last)); }
"sample_02", 350L); assertEquals(3, fileList.getTaskCount()); assertEquals("sample_00", fileList.get(0).get(0)); assertEquals("sample_01", fileList.get(1).get(0)); assertEquals("sample_02", fileList.get(2).get(0)); "sample_02", 350L); assertEquals(3, fileList.getTaskCount()); assertEquals("sample_00", fileList.get(0).get(0)); assertEquals("sample_01", fileList.get(1).get(0)); assertEquals("sample_02", fileList.get(2).get(0)); "sample_02", 350L); assertEquals(2, fileList.getTaskCount()); assertEquals("sample_00", fileList.get(0).get(0)); assertEquals("sample_01", fileList.get(0).get(1)); assertEquals("sample_02", fileList.get(1).get(0)); "sample_02", 350L); assertEquals(1, fileList.getTaskCount()); assertEquals("sample_00", fileList.get(0).get(0)); assertEquals("sample_01", fileList.get(0).get(1)); assertEquals("sample_02", fileList.get(0).get(2));
@Override public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { PluginTask task = config.loadConfig(getTaskClass()); validateInputTask(task); // list files recursively task.setFiles(listFiles(task)); // number of processors is same with number of files return resume(task.dump(), task.getFiles().getTaskCount(), control); }
@Override public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(getTaskClass()); // validate task newS3Client(task); control.run(taskSource, taskCount); // build next config ConfigDiff configDiff = Exec.newConfigDiff(); // last_path if (task.getIncremental()) { Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath()); LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse("")); configDiff.set("last_path", lastPath); } return configDiff; }