ObjectListing listing = s3.listObjects( bucketName, prefix ); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3.listNextBatchOfObjects (listing); summaries.addAll (listing.getObjectSummaries()); }
private static List<String> listByFullPathPrefix(final String bucket, final String s3prefix) { return AmazonS3ClientBuilder.defaultClient().listObjects(bucket, s3prefix).getObjectSummaries() .stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()); }
private Iterator<LocatedFileStatus> statusFromListing(ObjectListing listing) { return Iterators.concat( statusFromPrefixes(listing.getCommonPrefixes()), statusFromObjects(listing.getObjectSummaries())); }
private Iterator<String> createIterator() { if (objectListing == null) initListing(); List<String> temp = Lists.newArrayList(); for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { temp.add(summary.getKey()); } return temp.iterator(); }
/** * Returns the objects names in a bucket. * * @param bucketName * the bucket whose contents will be listed. * @return the set of names of objects contained in a bucket. * @throws Exception */ public String[] getS3ObjectsNames( String bucketName ) throws Exception { Bucket bucket = getBucket( bucketName ); if ( bucket == null ) { throw new Exception( Messages.getString( "S3DefaultService.Exception.UnableToFindBucket.Message", bucketName ) ); } return getS3Objects( bucket ).getObjectSummaries().stream().map( b -> b.getKey() ).toArray( String[]::new ); }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
List<S3ObjectSummary> summaries = listing.getObjectSummaries(); List<String> keyList = new ArrayList(); int startkeyNumber = 0; summaries.addAll(listing.getObjectSummaries());
ObjectListing currentList = s3.listObjects(bucket, prefix); while(true){ for(S3ObjectSummary obj:currentList.getObjectSummaries()) try { succ.add(S3FileVec.make(obj,fs).toString());
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
@Override protected List<String> getFileNames(String lockPrefix) throws Exception { ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucket); request.setPrefix(lockPrefix); ObjectListing objectListing = client.listObjects(request); return Lists.transform ( objectListing.getObjectSummaries(), new Function<S3ObjectSummary, String>() { @Override public String apply(S3ObjectSummary summary) { return summary.getKey(); } } ); } }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
listing.getObjectSummaries(), new Predicate<S3ObjectSummary>()
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary)iterator.next();
if (!objectListing.getObjectSummaries().isEmpty()) { nextMarker = objectListing.getObjectSummaries() .get(objectListing.getObjectSummaries().size() - 1) .getKey(); objectListing.getObjectSummaries().add(currentObject); currentObject = null;
public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail){ for( S3ObjectSummary obj : listing.getObjectSummaries() ) { try { Key k = PersistS3.loadKey(obj); JsonObject o = new JsonObject(); o.addProperty(KEY, k.toString()); o.addProperty(FILE, obj.getKey()); o.addProperty(VALUE_SIZE, obj.getSize()); succ.add(o); } catch( IOException e ) { JsonObject o = new JsonObject(); o.addProperty(FILE, obj.getKey()); o.addProperty(ERROR, e.getMessage()); fail.add(o); } } }
private void prepareCurrentListing() { while (currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated())) { if (currentListing == null) { ListObjectsRequest req = new ListObjectsRequest(); req.setBucketName(getBucketName()); req.setPrefix(getPrefix()); req.setMaxKeys(getBatchSize()); req.setDelimiter(getDelimiter()); req.setMarker(getMarker()); req.setEncodingType(getEncodingType()); req.setRequesterPays(isRequesterPays()); currentListing = getS3().listObjects(req); } else { currentListing = getS3().listNextBatchOfObjects(currentListing); } currentIterator = currentListing.getObjectSummaries().iterator(); } }
@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
for (S3ObjectSummary sum : list.getObjectSummaries()) { S3CheckpointData data = read(sum.getKey());