@Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } };
@Override public boolean hasNext() { if (iterator.hasNext()) { return true; } else { while (objectListing.isTruncated() && !iterator.hasNext()) { objectListing = s3Client.listNextBatchOfObjects(objectListing); iterator = createIterator(); } } return iterator.hasNext(); }
currentList = s3.listNextBatchOfObjects(currentList); else break;
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
object_listing = s3.listNextBatchOfObjects(object_listing); } else { break;
listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse);
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Override protected Response serve() { String bucket = _bucket.value(); Log.info("ImportS3 processing (" + bucket + ")"); JsonObject json = new JsonObject(); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); AmazonS3 s3 = PersistS3.getClient(); ObjectListing currentList = s3.listObjects(bucket); processListing(currentList, succ, fail); while(currentList.isTruncated()){ currentList = s3.listNextBatchOfObjects(currentList); processListing(currentList, succ, fail); } json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); DKV.write_barrier(); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); return r; } }
list = s3.listNextBatchOfObjects(list); else break;
@Test public void testGetsNextPageWithParamsFromFirstPage() throws Exception { when(objectListing.isTruncated()).thenReturn(true); ObjectListing secondPage = mock(ObjectListing.class); when(secondPage.isTruncated()).thenReturn(false); when(s3.listNextBatchOfObjects(objectListing)).thenReturn(secondPage); s3Objects.iterator().hasNext(); verify(s3).listNextBatchOfObjects(objectListing); }
bucketListing = amazonS3.listNextBatchOfObjects(bucketListing); summaries.addAll(bucketListing.getObjectSummaries());
@Test public void testReturnsObjectSummariesFromMultiplePages() throws Exception { when(objectListing.isTruncated()).thenReturn(true); when(objectListing.getObjectSummaries()).thenReturn( Arrays.asList(firstSummary)); ObjectListing secondPage = mock(ObjectListing.class); when(secondPage.isTruncated()).thenReturn(true); when(secondPage.getObjectSummaries()).thenReturn( new ArrayList<S3ObjectSummary>()); when(s3.listNextBatchOfObjects(objectListing)).thenReturn(secondPage); ObjectListing thirdPage = mock(ObjectListing.class); when(thirdPage.isTruncated()).thenReturn(false); when(thirdPage.getObjectSummaries()).thenReturn( Arrays.asList(secondSummary)); when(s3.listNextBatchOfObjects(secondPage)).thenReturn(thirdPage); Iterator<S3ObjectSummary> iter = s3Objects.iterator(); assertTrue(iter.hasNext()); assertSame(firstSummary, iter.next()); assertTrue(iter.hasNext()); assertSame(secondSummary, iter.next()); assertFalse(iter.hasNext()); }
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse);
/** * List the next set of objects. * @param objects paged result * @return the next result object */ protected ObjectListing continueListObjects(ObjectListing objects) { return mClient.listNextBatchOfObjects(objects); }
/** * List the next set of objects. * @param objects paged result * @return the next result object */ protected ObjectListing continueListObjects(ObjectListing objects) { return mClient.listNextBatchOfObjects(objects); }
@Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3Client.listNextBatchOfObjects(previous); } };
@Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3Client.listNextBatchOfObjects(previous); } };
private List<S3ObjectSummary> getS3ObjectSummaries(String prefix) { // S3 limits the size of the response to 1000 entries. Batch the requests. ObjectListing listing = s3.listObjects(bucket, prefix); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); } return summaries; }
private void copy(AmazonS3URI source, AmazonS3URI target) { ListObjectsRequest request = listObjectsRequestFactory .newInstance() .withBucketName(source.getBucket()) .withPrefix(source.getKey()); ObjectListing listing = srcClient.listObjects(request); submitCopyJobsFromListing(source, target, request, listing); while (listing.isTruncated()) { listing = srcClient.listNextBatchOfObjects(listing); submitCopyJobsFromListing(source, target, request, listing); } }