private static List<String> listByFullPathPrefix(final String bucket, final String s3prefix) { return AmazonS3ClientBuilder.defaultClient().listObjects(bucket, s3prefix).getObjectSummaries() .stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()); }
/** * Returns the objects in a bucket. The objects returned by this method contain only minimal information. * * @param bucket * the bucket whose contents will be listed. * @return the set of objects contained in a bucket. * @throws SdkClientException */ private ObjectListing getS3Objects( Bucket bucket ) throws SdkClientException { ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader() ); return s3Client.listObjects( bucket.getName() ); } finally { Thread.currentThread().setContextClassLoader( currentClassLoader ); } }
ArrayList<String> succ = new ArrayList<String>(); ArrayList<String> fail = new ArrayList<String>(); ObjectListing currentList = s3.listObjects(bucket, prefix); while(true){ for(S3ObjectSummary obj:currentList.getObjectSummaries())
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
private void initListing() { ListObjectsRequest listReq = new ListObjectsRequest(); listReq.setBucketName(bucket); listReq.setPrefix(prefix); if (StringUtils.isNotBlank(delimiter)) listReq.setDelimiter(delimiter); if (StringUtils.isNotBlank(marker)) listReq.setMarker(marker); objectListing = s3Client.listObjects(listReq); }
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
ObjectListing object_listing = s3.listObjects(bucket_name); while (true) { for (Iterator<?> iterator =
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse);
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
@Override protected Response serve() { String bucket = _bucket.value(); Log.info("ImportS3 processing (" + bucket + ")"); JsonObject json = new JsonObject(); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); AmazonS3 s3 = PersistS3.getClient(); ObjectListing currentList = s3.listObjects(bucket); processListing(currentList, succ, fail); while(currentList.isTruncated()){ currentList = s3.listNextBatchOfObjects(currentList); processListing(currentList, succ, fail); } json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); DKV.write_barrier(); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); return r; } }
ObjectListing list = s3.listObjects(bucketName);
private String resolveSingularPath(ObjectType objectType, String objectKey, AwsNamedAccountCredentials credentials, AmazonS3 amazonS3, String bucket) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; ObjectListing bucketListing = amazonS3.listObjects( new ListObjectsRequest(bucket, rootFolder, null, null, 10000) ); List<S3ObjectSummary> summaries = bucketListing.getObjectSummaries(); if (summaries != null && summaries.size() == 1) { return summaries.get(0).getKey(); } else { throw new IllegalArgumentException("Unable to resolve singular " + objectType + " at " + daoRoot(credentials, objectType.getGroup()) + '/' + objectKey + "."); } }
@Before public void setUpSharedTests() throws Exception { objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn( new ArrayList<S3ObjectSummary>()); when(objectListing.isTruncated()).thenReturn(false); s3 = mock(AmazonS3.class); when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn( objectListing); firstSummary = mock(S3ObjectSummary.class); secondSummary = mock(S3ObjectSummary.class); }
@Test public void testUsesBatchSizeForMaxKeys() throws Exception { s3Objects.withBatchSize(12).iterator().hasNext(); ArgumentCaptor<ListObjectsRequest> listCaptor = ArgumentCaptor .forClass(ListObjectsRequest.class); verify(s3).listObjects(listCaptor.capture()); assertEquals((Integer) 12, listCaptor.getValue().getMaxKeys()); }
@Test public void testReturnsObjectSummariesFromFirstPage() throws Exception { when(objectListing.getObjectSummaries()).thenReturn( Arrays.asList(firstSummary, secondSummary)); Iterator<S3ObjectSummary> iter = s3Objects.iterator(); assertTrue(iter.hasNext()); assertSame(firstSummary, iter.next()); assertTrue(iter.hasNext()); assertSame(secondSummary, iter.next()); assertFalse(iter.hasNext()); verify(s3, times(1)).listObjects(any(ListObjectsRequest.class)); }
@Test public void testCallsListObjects() throws Exception { s3Objects.iterator().hasNext(); ArgumentCaptor<ListObjectsRequest> listCaptor = ArgumentCaptor .forClass(ListObjectsRequest.class); verify(s3).listObjects(listCaptor.capture()); assertEquals(s3Objects.getBucketName(), listCaptor.getValue().getBucketName()); assertEquals(s3Objects.getPrefix(), listCaptor.getValue().getPrefix()); assertNull(listCaptor.getValue().getMaxKeys()); }