@Override public void setBucketName(String bucketName) { listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); }
private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest);
@Override @Nullable protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException { String delimiter = recursive ? "" : PATH_SEPARATOR; key = PathUtils.normalizePath(key, PATH_SEPARATOR); // In case key is root (empty string) do not normalize prefix. key = key.equals(PATH_SEPARATOR) ? "" : key; if (mConf.isSet(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1) && mConf .get(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1).equals(Boolean.toString(true))) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ObjectListing result = getObjectListingChunkV1(request); if (result != null) { return new S3AObjectListingChunkV1(request, result); } } else { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ListObjectsV2Result result = getObjectListingChunk(request); if (result != null) { return new S3AObjectListingChunk(request, result); } } return null; }
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) { LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName)); List<S3ObjectSummary> objects = new ArrayList<>(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.withBucketName(bucketName); listObjectsRequest.withPrefix(directory); ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); if(ol.isTruncated()) { do { objects.addAll(ol.getObjectSummaries()); listObjectsRequest.setMarker(ol.getNextMarker()); ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest); } while (ol.isTruncated()); } else { objects.addAll(ol.getObjectSummaries()); } if (objects.isEmpty()) { return emptyList(); } return unmodifiableList(objects); }
if (listObjectsResponse == null) { final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest);
@Override public void setBucketName(String bucketName) { listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); }
private void registerDeleteObject(String key, String bucket) { if (policy.shouldDelay(key)) { // Record summary so we can add it back for some time post-deletion ListObjectsRequest request = new ListObjectsRequest() .withBucketName(bucket) .withPrefix(key); S3ObjectSummary summary = innerlistObjects(request).getObjectSummaries() .stream() .filter(result -> result.getKey().equals(key)) .findFirst() .orElse(null); delayedDeletes.put(key, new Delete(System.currentTimeMillis(), summary)); } }
private boolean fileExistsinS3(String bucketName, String path) { return s3Client.listObjects(new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(path.substring(1)) .withMaxKeys(1)).getObjectSummaries().isEmpty(); }
public long fetchLastCommittedOffset() { logger.debug("Getting max offset for " + topic + ":" + partition); String prefix = getKeyPrefix(); logger.debug("Listing keys for bucket/prefix " + bucket + "/" + prefix); List<S3ObjectSummary> objectSummaries = awsClient.listObjects(new ListObjectsRequest().withBucketName(bucket).withDelimiter("/").withPrefix(prefix)).getObjectSummaries(); logger.debug("Received result " + objectSummaries); long maxOffset = 0; for (S3ObjectSummary objectSummary : objectSummaries) { logger.debug(objectSummary.getKey()); String[] offsets = objectSummary.getKey().substring(prefix.length()).split("_"); long endOffset = Long.valueOf(offsets[1]); if (endOffset > maxOffset) maxOffset = endOffset; } return maxOffset; }
@Override public ObjectListing call() { ListObjectsRequest request = new ListObjectsRequest().withBucketName(config.getBucketName()); request.setPrefix("".equals(prefix) ? null : prefix); // Note: AWS SDK will always set encoding-type=url, but will only decode automatically if we // leave the value null return s3.listObjects(request); } }, OPERATION_LIST_OBJECTS);
@RequestMapping(method = RequestMethod.GET) public List<Resource<S3ObjectSummary>> getBucketResources() { ObjectListing objectListing = amazonS3Template.getAmazonS3Client() .listObjects(new ListObjectsRequest() .withBucketName(bucketName)); return objectListing.getObjectSummaries() .stream() .map(a -> new Resource<>(a, new Link(String.format("https://s3.amazonaws.com/%s/%s", a.getBucketName(), a.getKey())).withRel("url"))) .collect(Collectors.toList()); }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withDelimiter("/") .withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add); listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (final AmazonServiceException e) { log.error("Could not list common prefixes in S3", e); } return commonPrefixes; }
@Override protected void doStart() { AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3 s3Client = new AmazonS3Client(myCredentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ChannelProcessor channelProcessor = getChannelProcessor(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { String file = s3ObjectSummary.getKey(); LOGGER.info("Read the content of {}", file); GetObjectRequest objectRequest = new GetObjectRequest(bucket, file); S3Object objectPortion = s3Client.getObject(objectRequest); try { long startTime = System.currentTimeMillis(); processLines(channelProcessor, objectPortion.getObjectContent()); LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime); } catch (IOException e) { LOGGER.warn("Cannot process the {}, skipping", file, e); } } }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
/*** * Get list of S3 objects within a S3 bucket qualified by prefix path * * @param bucketName S3 bucket name * @param prefix S3 prefix to object * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path */ public List<S3ObjectSummary> listS3Bucket(String bucketName, String prefix) { final AmazonS3 amazonS3 = getS3Client(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(prefix); final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest); LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing); return objectListing.getObjectSummaries(); }
public List<S3ObjectSummary> listFiles(String filePrefix) { ListObjectsRequest listObjectRequest = new ListObjectsRequest().withBucketName(getAmazonType().getS3Bucket()) .withPrefix(filePrefix); return getAmazonType().getAmazonS3().listObjects(listObjectRequest).getObjectSummaries(); }
private void copy(AmazonS3URI source, AmazonS3URI target) { ListObjectsRequest request = listObjectsRequestFactory .newInstance() .withBucketName(source.getBucket()) .withPrefix(source.getKey()); ObjectListing listing = srcClient.listObjects(request); submitCopyJobsFromListing(source, target, request, listing); while (listing.isTruncated()) { listing = srcClient.listNextBatchOfObjects(listing); submitCopyJobsFromListing(source, target, request, listing); } }
private void copy(AmazonS3URI source, AmazonS3URI target) { ListObjectsRequest request = listObjectsRequestFactory .newInstance() .withBucketName(source.getBucket()) .withPrefix(source.getKey()); ObjectListing listing = srcClient.listObjects(request); submitCopyJobsFromListing(source, target, request, listing); while (listing.isTruncated()) { listing = srcClient.listNextBatchOfObjects(listing); submitCopyJobsFromListing(source, target, request, listing); } }