public S3FileInput(PluginTask task, int taskIndex) { super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex)); }
public boolean needsMore() { return size() < limitCount; }
public SingleFileProvider(PluginTask task, int taskIndex) { this.client = newS3Client(task); this.bucket = task.getBucket(); this.iterator = task.getFiles().get(taskIndex).iterator(); this.retryExec = retryExecutorFrom(task); }
@Override public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { PluginTask task = config.loadConfig(getTaskClass()); validateInputTask(task); // list files recursively task.setFiles(listFiles(task)); // number of processors is same with number of files return resume(task.dump(), task.getFiles().getTaskCount(), control); }
private static FileList newFileList(ConfigSource config, Object... nameAndSize) { FileList.Builder builder = new FileList.Builder(config); for (int i = 0; i < nameAndSize.length; i += 2) { builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]); } return builder.build(); } }
@Test(expected = Deny.class) @SuppressWarnings("unchecked") public void execute_should_unwrap_RetryGiveupException() throws Exception { new DefaultRetryable(Deny.until(4)) .executeWith(retryExecutor().withRetryLimit(3)); }
private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize) { FileList.Builder builder = new FileList.Builder(config); for (int i = 0; i < nameAndSize.length; i += 2) { builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]); } return builder; }
@Test(expected = RetryGiveupException.class) @SuppressWarnings("unchecked") public void fail_after_exceeding_attempts_just_like_Retryable() throws Exception { retryExecutor() .withRetryLimit(3) .run(new DefaultRetryable(Deny.until(4))); }
public Builder(Task task) { this(); this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern()); this.limitCount = task.getTotalFileCountLimit(); this.minTaskSize = task.getMinTaskSize(); }
@Test(expected = IllegalStateException.class) public void execute_without_an_implementation_should_throw_an_IllegalStateException() { new DefaultRetryable().executeWith(retryExecutor()); } }
/** * Build the common retry executor from some configuration params of plugin task. * @param task Plugin task. * @return RetryExecutor object */ private static RetryExecutor retryExecutorFrom(RetrySupportPluginTask task) { return retryExecutor() .withRetryLimit(task.getMaximumRetries()) .withInitialRetryWait(task.getInitialRetryIntervalMillis()) .withMaxRetryWait(task.getMaximumRetryIntervalMillis()); }
/** * A base builder for the subclasses to then customize.builder * @param task Embulk plugin * @return AmazonS3 client b **/ protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task) { return AmazonS3ClientBuilder .standard() .withCredentials(getCredentialsProvider(task)) .withClientConfiguration(getClientConfiguration(task)); }
@Override public TransactionalFileInput open(TaskSource taskSource, int taskIndex) { PluginTask task = taskSource.loadTask(getTaskClass()); return new S3FileInput(task, taskIndex); }
@VisibleForTesting public static void listS3FilesByPrefix(FileList.Builder builder, final AmazonS3 client, String bucketName, String prefix, Optional<String> lastPath, boolean skipGlacierObjects) { listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null); }
@Override protected ClientConfiguration getClientConfiguration(PluginTask task) { RiakCsPluginTask t = (RiakCsPluginTask) task; ClientConfiguration config = super.getClientConfiguration(t); config.setSignerOverride("S3SignerType"); return config; } }
/** * Provide an overridable default client. * Since this returns an immutable object, it is not for any further customizations by mutating, * e.g., {@link AmazonS3#setEndpoint} will throw a runtime {@link UnsupportedOperationException} * Subclass's customization should be done through {@link AbstractS3FileInputPlugin#defaultS3ClientBuilder}. * @param task Embulk plugin task * @return AmazonS3 */ protected AmazonS3 newS3Client(PluginTask task) { return defaultS3ClientBuilder(task).build(); }
private String readNextString() { return new String(readNext(), StandardCharsets.UTF_8); } }
static Deny until(int calls) { return new Deny(calls); }
@Test(expected = RuntimeException.class) @SuppressWarnings("unchecked") public void execute_should_unwrap_RetryGiveupException_but_rewrap_checked_exception_in_a_RuntimeException() { new DefaultRetryable(Deny.until(4).with(new Exception("A checked exception"))) .executeWith(retryExecutor().withRetryLimit(3)); }
@Test @SuppressWarnings("unchecked") public void guarantee_retry_attempts_just_like_Retryable() throws Exception { retryExecutor() .withRetryLimit(0) .run(new DefaultRetryable(Deny.until(0))); retryExecutor() .withRetryLimit(1) .run(new DefaultRetryable(Deny.until(1))); retryExecutor() .withRetryLimit(2) .run(new DefaultRetryable(Deny.until(1))); retryExecutor() .withRetryLimit(3) .run(new DefaultRetryable(Deny.until(2))); }