AmazonS3 client = newS3Client(task); String bucketName = task.getBucket(); FileList.Builder builder = new FileList.Builder(task); RetryExecutor retryExec = retryExecutorFrom(task); if (task.getPath().isPresent()) { LOGGER.info("Found total [{}] files", builder.size()); return builder.build();
@VisibleForTesting public void addS3DirectObject(FileList.Builder builder, final AmazonS3 client, String bucket, String objectKey, RetryExecutor retryExec) { final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey); ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") { @Override public ObjectMetadata call() { return client.getObjectMetadata(objectMetadataRequest); } }.executeWith(retryExec); builder.add(objectKey, objectMetadata.getContentLength()); }
public FileList build() { try { stream.close(); } catch (IOException ex) { throw Throwables.propagate(ex); } return new FileList(binary.toByteArray(), getSplits(entries), Optional.ofNullable(last)); }
public synchronized boolean add(String path, long size) { // TODO throw IllegalStateException if stream is already closed if (!needsMore()) { return false; } if (!pathMatchPattern.matcher(path).find()) { return false; } int index = entries.size(); entries.add(new Entry(index, size)); byte[] data = path.getBytes(StandardCharsets.UTF_8); castBuffer.putInt(0, data.length); try { stream.write(castBuffer.array()); stream.write(data); } catch (IOException ex) { throw Throwables.propagate(ex); } last = path; return true; }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code() { AmazonServiceException exception = new AmazonServiceException("expired token exception"); exception.setStatusCode(HttpStatus.SC_BAD_REQUEST); exception.setErrorCode("ExpiredToken"); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code() { AmazonServiceException exception = new AmazonServiceException("method not allow exception"); exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
public boolean needsMore() { return size() < limitCount; }
@Test public void listS3FilesByPrefix() { doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true); }
private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize) { FileList.Builder builder = new FileList.Builder(config); for (int i = 0; i < nameAndSize.length; i += 2) { builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]); } return builder; }
@Test(expected = SomeException.class) public void addS3DirectObject_on_retry_gave_up_should_throw_original_exception() { doThrow(new SomeException()).doReturn(new ObjectMetadata()) .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject( builder, client, "some_bucket", "some_prefix", retryExecutor().withRetryLimit(0)); } }
@Test(expected = SomeException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception() { doThrow(new SomeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(0)); }
@Test public void addS3DirectObject_with_retry() { doThrow(new RuntimeException()).doReturn(new ObjectMetadata()) .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject( builder, client, "some_bucket", "some_prefix", retryExecutor()); }
private static FileList newFileList(ConfigSource config, Object... nameAndSize) { FileList.Builder builder = new FileList.Builder(config); for (int i = 0; i < nameAndSize.length; i += 2) { builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]); } return builder.build(); } }
@Test public void addS3DirectObject() { doReturn(new ObjectMetadata()).when(client).getObjectMetadata(any(GetObjectMetadataRequest.class)); FileList.Builder builder = new FileList.Builder().pathMatchPattern(""); dummyS3Plugin().addS3DirectObject(builder, client, "some_bucket", "some_prefix"); }
@Test(expected = AmazonServiceException.class) public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code() { AmazonServiceException exception = new AmazonServiceException("Forbidden exception"); exception.setStatusCode(HttpStatus.SC_FORBIDDEN); exception.setErrorType(AmazonServiceException.ErrorType.Client); doThrow(exception).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }
@Test public void listS3FileByPrefix_with_retry() { doThrow(new RuntimeException()).doReturn(new ObjectListing()) .when(client).listObjects(any(ListObjectsRequest.class)); FileList.Builder builder = new FileList.Builder(); dummyS3Plugin().listS3FilesByPrefix( builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true, retryExecutor().withRetryLimit(1)); }