@VisibleForTesting public static void listS3FilesByPrefix(FileList.Builder builder, final AmazonS3 client, String bucketName, String prefix, Optional<String> lastPath, boolean skipGlacierObjects) { listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null); }
@Test(expected = ConfigException.class) public void useSkipGlacierObjects() throws Exception { AmazonS3 client; client = mock(AmazonS3.class); doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class)); AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS); plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.empty(), false); }
@Test public void listS3FilesByPrefix() { doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true); }
listS3FilesByPrefix(builder, client, bucketName, task.getPathPrefix().get(), task.getLastPath(), task.getSkipGlacierObjects(), retryExec); LOGGER.info("Found total [{}] files", builder.size());
@Test public void listS3FileByPrefix_with_retry() { doThrow(new RuntimeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test(expected = SomeException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception() { doThrow(new SomeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(0)); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code() { AmazonServiceException exception = new AmazonServiceException("method not allow exception"); exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code() { AmazonServiceException exception = new AmazonServiceException("Forbidden exception"); exception.setStatusCode(HttpStatus.SC_FORBIDDEN); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code() { AmazonServiceException exception = new AmazonServiceException("expired token exception"); exception.setStatusCode(HttpStatus.SC_BAD_REQUEST); exception.setErrorCode("ExpiredToken"); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }