public SingleFileProvider(PluginTask task, int taskIndex) { this.client = newS3Client(task); this.bucket = task.getBucket(); this.iterator = task.getFiles().get(taskIndex).iterator(); this.retryExec = retryExecutorFrom(task); }
@Override public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(getTaskClass()); // validate task newS3Client(task); control.run(taskSource, taskCount); // build next config ConfigDiff configDiff = Exec.newConfigDiff(); // last_path if (task.getIncremental()) { Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath()); LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse("")); configDiff.set("last_path", lastPath); } return configDiff; }
private FileList listFiles(final PluginTask task) AmazonS3 client = newS3Client(task); String bucketName = task.getBucket(); FileList.Builder builder = new FileList.Builder(task);