@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws SdkClientException, AmazonServiceException { return listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(previousObjectListing)); }
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws Exception { RefCountedClient holder = client.get(); AmazonS3Client amazonS3Client = holder.useClient(); try { return amazonS3Client.listNextBatchOfObjects(previousObjectListing); } finally { holder.release(); } }
listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries());
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws AmazonClientException, AmazonServiceException { assertParameterNotNull( previousObjectListing, "The previous object listing parameter must be specified when listing the next batch of objects in a bucket"); return listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(previousObjectListing)); }
/** * Get next batch of S3 objects * * @param objectListing instance of {@link ObjectListing} used in previous listObject call * @return the next set of S3 keys */ public ObjectListing listObjects(ObjectListing objectListing) { return s3Client.listNextBatchOfObjects(objectListing); } }
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws SdkClientException, AmazonServiceException { return listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(previousObjectListing)); }
@Override public ObjectListing listNextBatchOfObjects(ObjectListing previousObjectListing) throws AmazonClientException, AmazonServiceException { assertParameterNotNull( previousObjectListing, "The previous object listing parameter must be specified when listing the next batch of objects in a bucket"); return listNextBatchOfObjects(new ListNextBatchOfObjectsRequest(previousObjectListing)); }
private ObjectListing s3getNextBatch() { final MirrorOptions options = context.getOptions(); final boolean verbose = options.isVerbose(); final int maxRetries = options.getMaxRetries(); for (int tries=0; tries<maxRetries; tries++) { try { context.getStats().s3getCount.incrementAndGet(); ObjectListing next = client.listNextBatchOfObjects(listing); if (verbose) log.info("successfully got next batch of objects (on try #"+tries+")"); return next; } catch (AmazonS3Exception s3e) { log.error("s3 exception listing objects (try #"+tries+"): "+s3e); } catch (Exception e) { log.error("unexpected exception listing objects (try #"+tries+"): "+e); } if (Sleep.sleep(50)) { log.info("s3getNextBatch: interrupted while waiting for next try"); break; } } throw new IllegalStateException("Too many errors trying to list objects (maxRetries="+maxRetries+")"); }
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);
BasicAWSCredentials credentials = new BasicAWSCredentials("accessKey", "secretAccessKey"); AmazonS3Client client = new AmazonS3Client(credentials); for (ObjectListing listing = client.listObjects("loggingBucket");; listing = client.listNextBatchOfObjects(listing)) { // run through each listing to process it for (S3ObjectSummary summary : listing.getObjectSummaries()) { String bucketName = summary.getBucketName(); String fileName = summary.getKey(); S3Object object = client.getObject(bucketName, fileName); S3ObjectInputStream stream = object.getObjectContent(); BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); ... process the file here writing it locally or something // delete the file afterwards client.deleteObject(bucketName, fileName); } // we may need to get more files if (!listing.isTruncated()) { break; } }
public void execute() { AmazonS3Client client = getOrCreateClient(AmazonS3Client.class); if (key != null) { File targetFile = file == null ? new File(key) : file; downloadObjectToFile(client, targetFile, key); } else { ObjectListing objectListing = client.listObjects(bucketName); while (true) { for (Iterator<?> iterator = objectListing.getObjectSummaries() .iterator(); iterator.hasNext();) { S3ObjectSummary objectSummary = (S3ObjectSummary) iterator .next(); String key = objectSummary.getKey(); if (key.startsWith(keyPrefix)) { downloadObjectToFile(client, new File(dir + File.pathSeparator + key), key); } } if (objectListing.isTruncated()) { objectListing = client .listNextBatchOfObjects(objectListing); } else { break; } } } } }
public BucketPrefixSummary summarize(AmazonS3Client client, String bucketName) { BucketPrefixSummary summary = new BucketPrefixSummary(); ListObjectsRequest request = getListObjectsRequest(bucketName); ObjectListing current = client.listObjects(request); summarize(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); summarize(summary, current.getObjectSummaries()); } log.debug("Completed summary for '{}'", bucketName); return summary; }
public BucketPrefixSummary summarize(AmazonS3Client client, String bucketName) { BucketPrefixSummary summary = new BucketPrefixSummary(); ListObjectsRequest request = getListObjectsRequest(bucketName); ObjectListing current = client.listObjects(request); summarize(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); summarize(summary, current.getObjectSummaries()); } log.debug("Completed summary for '{}'", bucketName); return summary; }
public BucketPrefixSummary summarize(AmazonS3Client client, String bucketName, BucketPrefixSummary summary) { ListObjectsRequest request = getListObjectsRequest(bucketName, summary.getPrefix()); ObjectListing current = client.listObjects(request); summarize(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); summarize(summary, current.getObjectSummaries()); } log.debug("Completed summary for prefix '{}'", summary.getPrefix()); return summary; }
public BucketPrefixSummary summarize(AmazonS3Client client, String bucketName, BucketPrefixSummary summary) { ListObjectsRequest request = getListObjectsRequest(bucketName, summary.getPrefix()); ObjectListing current = client.listObjects(request); summarize(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); summarize(summary, current.getObjectSummaries()); } log.debug("Completed summary for prefix '{}'", summary.getPrefix()); return summary; }
public BucketSummary getBucketSummary(AmazonS3Client client, Bucket bucket) { BucketSummary summary = new BucketSummary(); summary.setBucket(bucket); ListObjectsRequest request = getListObjectsRequest(bucket.getName()); ObjectListing current = client.listObjects(request); updateBucketSummary(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); updateBucketSummary(summary, current.getObjectSummaries()); } log.debug("Completed summary for '{}'", bucket.getName()); return summary; }
@Test public void testGetAllBinaryKeys() throws BinaryStoreException { List<S3ObjectSummary> objectList = new ArrayList<>(); for(int i=0; i< 101; i++) { S3ObjectSummary object = new S3ObjectSummary(); object.setKey(String.valueOf(i)); objectList.add(object); } // First request to get objects (incomplete list returned) expect(s3Client.listObjects(isA(ListObjectsRequest.class))) .andReturn(objectListing); expect(objectListing.getObjectSummaries()).andReturn(objectList); expect(objectListing.isTruncated()).andReturn(true); // Second request to get more objects expect(s3Client.listNextBatchOfObjects(objectListing)) .andReturn(objectListing); expect(objectListing.getObjectSummaries()).andReturn(objectList); expect(objectListing.isTruncated()).andReturn(false); replayAll(); Iterable<BinaryKey> allKeys = s3BinaryStore.getAllBinaryKeys(); int keyCount = 0; for(BinaryKey key : allKeys) { keyCount++; } assertEquals(202, keyCount); // Expecting two sets of 101 objects }
public BucketSummary getBucketSummary(AmazonS3Client client, Bucket bucket) { BucketSummary summary = new BucketSummary(); summary.setBucket(bucket); ListObjectsRequest request = getListObjectsRequest(bucket.getName()); ObjectListing current = client.listObjects(request); updateBucketSummary(summary, current.getObjectSummaries()); while (current.isTruncated()) { current = client.listNextBatchOfObjects(current); updateBucketSummary(summary, current.getObjectSummaries()); } log.debug("Completed summary for '{}'", bucket.getName()); return summary; }
public static void emptyAndDeleteBucket(AmazonS3Client client, String bucketName) { ObjectListing objectListing = client.listObjects(bucketName); while (true) { for (Iterator<?> iterator = objectListing.getObjectSummaries() .iterator(); iterator.hasNext();) { S3ObjectSummary objectSummary = (S3ObjectSummary) iterator .next(); client.deleteObject(bucketName, objectSummary.getKey()); } if (objectListing.isTruncated()) { objectListing = client.listNextBatchOfObjects(objectListing); } else { break; } } client.deleteBucket(bucketName); }
prevObjectListing = s3service.listNextBatchOfObjects(prevObjectListing);