@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SdkClientException, AmazonServiceException { listObjectsRequest = beforeClientExecution(listObjectsRequest); rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag is enabled * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)}, * otherwise, it will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null; Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "ListObjects"); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { assertParameterNotNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag * is enabled only when the customers don't explicitly call * {@link ListObjectsRequest#setEncodingType(String)}, otherwise, it * will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = Constants.URL_ENCODING .equals(listObjectsRequest.getEncodingType()); // if the url is encoded, the prefix also has to be encoded. final Request<ListObjectsRequest> request = createRequest( listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "encoding-type", listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) { request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); } return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName); ObjectListing listing = null; while((listing == null) || (request.getMarker() != null)) { listing = s3Client.listObjects(request); // do stuff with listing request.setMarker(listing.getNextMarker()); }
/** * Reads next key from S3 * @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to read */ public S3ObjectSummary getNextKey() { if (objectListing == null) { LOG.debug("Listing objects"); objectListing = s3Client.listObjects(listObjectsRequest); } // we have more elements in list if (currentPosition < objectListing.getObjectSummaries().size()) { currentObject = objectListing.getObjectSummaries().get(currentPosition++); } else if (objectListing.isTruncated()) { LOG.debug("Current batch reached its end. Fetching next page."); // when we're at the end, check if there's more data to read listObjectsRequest.setMarker(objectListing.getNextMarker()); LOG.debug("New marker is set to {}", listObjectsRequest.getMarker()); objectListing = s3Client.listObjects(listObjectsRequest); currentPosition = 0; if (currentPosition < objectListing.getObjectSummaries().size()) { currentObject = objectListing.getObjectSummaries().get(currentPosition++); } else { LOG.debug("No more objects to read"); currentObject = null; } } else { currentObject = null; } return currentObject; }
while (req.getMarker() != null);
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { assertParameterNotNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag * is enabled only when the customers don't explicitly call * {@link ListObjectsRequest#setEncodingType(String)}, otherwise, it * will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = Constants.URL_ENCODING .equals(listObjectsRequest.getEncodingType()); // if the url is encoded, the prefix also has to be encoded. final Request<ListObjectsRequest> request = createRequest( listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "encoding-type", listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) { request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); } return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }
@Override public ObjectListing answer(InvocationOnMock invocation) throws Throwable { ListObjectsRequest listObjectsRequest = invocation.getArgument(0); String marker = listObjectsRequest.getMarker(); ObjectListing objectListing = new ObjectListing(); if (marker == null) { objectListing.setNextMarker("marker"); objectListing.setTruncated(true); } else { assertEquals("marker", marker); S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setKey(expectedFilePath); s3ObjectSummary.setSize(expectedFileSize); objectListing.getObjectSummaries().add(s3ObjectSummary); } return objectListing; } });
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws SdkClientException, AmazonServiceException { listObjectsRequest = beforeClientExecution(listObjectsRequest); rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket"); /** * This flag shows whether we need to url decode S3 key names. This flag is enabled * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)}, * otherwise, it will be disabled for maintaining backwards compatibility. */ final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null; Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET); addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix()); addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker()); addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter()); if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString()); request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType()); populateRequesterPaysHeader(request, listObjectsRequest.isRequesterPays()); return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null); }