/** * Provide an overridable default client. * Since this returns an immutable object, it is not for any further customizations by mutating, * e.g., {@link AmazonS3#setEndpoint} will throw a runtime {@link UnsupportedOperationException} * Subclass's customization should be done through {@link AbstractS3FileInputPlugin#defaultS3ClientBuilder}. * @param task Embulk plugin task * @return AmazonS3 */ protected AmazonS3 newS3Client(PluginTask task) { return defaultS3ClientBuilder(task).build(); }
/** * A base builder for the subclasses to then customize.builder * @param task Embulk plugin * @return AmazonS3 client b **/ protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task) { return AmazonS3ClientBuilder .standard() .withCredentials(getCredentialsProvider(task)) .withClientConfiguration(getClientConfiguration(task)); }
@VisibleForTesting public static void listS3FilesByPrefix(FileList.Builder builder, final AmazonS3 client, String bucketName, String prefix, Optional<String> lastPath, boolean skipGlacierObjects) { listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null); }
@Override public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) { PluginTask task = config.loadConfig(getTaskClass()); validateInputTask(task); // list files recursively task.setFiles(listFiles(task)); // number of processors is same with number of files return resume(task.dump(), task.getFiles().getTaskCount(), control); }
private FileList listFiles(final PluginTask task) AmazonS3 client = newS3Client(task); String bucketName = task.getBucket(); FileList.Builder builder = new FileList.Builder(task); RetryExecutor retryExec = retryExecutorFrom(task); if (task.getPath().isPresent()) { LOGGER.info("Start getting object with path: [{}]", task.getPath().get()); addS3DirectObject(builder, client, task.getBucket(), task.getPath().get(), retryExec); listS3FilesByPrefix(builder, client, bucketName, task.getPathPrefix().get(), task.getLastPath(), task.getSkipGlacierObjects(), retryExec); LOGGER.info("Found total [{}] files", builder.size());
@VisibleForTesting public void addS3DirectObject(FileList.Builder builder, final AmazonS3 client, String bucket, String objectKey, RetryExecutor retryExec) { final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey); ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") { @Override public ObjectMetadata call() { return client.getObjectMetadata(objectMetadataRequest); } }.executeWith(retryExec); builder.add(objectKey, objectMetadata.getContentLength()); }
@Override protected ClientConfiguration getClientConfiguration(PluginTask task) { RiakCsPluginTask t = (RiakCsPluginTask) task; ClientConfiguration config = super.getClientConfiguration(t); config.setSignerOverride("S3SignerType"); return config; } }
@Override public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) { PluginTask task = taskSource.loadTask(getTaskClass()); // validate task newS3Client(task); control.run(taskSource, taskCount); // build next config ConfigDiff configDiff = Exec.newConfigDiff(); // last_path if (task.getIncremental()) { Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath()); LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse("")); configDiff.set("last_path", lastPath); } return configDiff; }
@Override public TransactionalFileInput open(TaskSource taskSource, int taskIndex) { PluginTask task = taskSource.loadTask(getTaskClass()); return new S3FileInput(task, taskIndex); }
public SingleFileProvider(PluginTask task, int taskIndex) { this.client = newS3Client(task); this.bucket = task.getBucket(); this.iterator = task.getFiles().get(taskIndex).iterator(); this.retryExec = retryExecutorFrom(task); }
@Test public void addS3DirectObject() { doReturn(new ObjectMetadata()).when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject(builder, client, "some_bucket", "some_prefix"); }
@Test(expected = ConfigException.class) public void useSkipGlacierObjects() throws Exception { AmazonS3 client; client = mock(AmazonS3.class); doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class)); AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS); plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.empty(), false); }
@Test public void addS3DirectObject_with_retry() { doThrow(new RuntimeException()).doReturn(new ObjectMetadata()) .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject( builder, client, "some_bucket", "some_prefix", retryExecutor()); }
@Override protected AmazonS3 newS3Client(PluginTask task) { RiakCsPluginTask t = (RiakCsPluginTask) task; return super .defaultS3ClientBuilder(task) .withEndpointConfiguration(new EndpointConfiguration(t.getEndpoint(), null)) .build(); }
@Test public void listS3FilesByPrefix() { doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true); }
@Test(expected = SomeException.class) public void addS3DirectObject_on_retry_gave_up_should_throw_original_exception() { doThrow(new SomeException()).doReturn(new ObjectMetadata()) .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject( builder, client, "some_bucket", "some_prefix", retryExecutor().withRetryLimit(0)); } }
Optional<String> region = t.getRegion(); AmazonS3ClientBuilder builder = super.defaultS3ClientBuilder(t);
@Test public void listS3FileByPrefix_with_retry() { doThrow(new RuntimeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test(expected = SomeException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception() { doThrow(new SomeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(0)); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code() { AmazonServiceException exception = new AmazonServiceException("method not allow exception"); exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }