private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
if ( listObjectsResponse == null ) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else {
@Override @Nullable protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException { String delimiter = recursive ? "" : PATH_SEPARATOR; key = PathUtils.normalizePath(key, PATH_SEPARATOR); // In case key is root (empty string) do not normalize prefix. key = key.equals(PATH_SEPARATOR) ? "" : key; if (mConf.isSet(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1) && mConf .get(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1).equals(Boolean.toString(true))) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ObjectListing result = getObjectListingChunkV1(request); if (result != null) { return new S3AObjectListingChunkV1(request, result); } } else { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ListObjectsV2Result result = getObjectListingChunk(request); if (result != null) { return new S3AObjectListingChunk(request, result); } } return null; }
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else {
public long fetchLastCommittedOffset() { logger.debug("Getting max offset for " + topic + ":" + partition); String prefix = getKeyPrefix(); logger.debug("Listing keys for bucket/prefix " + bucket + "/" + prefix); List<S3ObjectSummary> objectSummaries = awsClient.listObjects(new ListObjectsRequest().withBucketName(bucket).withDelimiter("/").withPrefix(prefix)).getObjectSummaries(); logger.debug("Received result " + objectSummaries); long maxOffset = 0; for (S3ObjectSummary objectSummary : objectSummaries) { logger.debug(objectSummary.getKey()); String[] offsets = objectSummary.getKey().substring(prefix.length()).split("_"); long endOffset = Long.valueOf(offsets[1]); if (endOffset > maxOffset) maxOffset = endOffset; } return maxOffset; }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withDelimiter("/") .withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add); listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (final AmazonServiceException e) { log.error("Could not list common prefixes in S3", e); } return commonPrefixes; }
private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += "/"; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(uri.getHost()) .withPrefix(key) .withDelimiter("/"); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
@Override protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException { List<String> directoryContents = new ArrayList<String>(); try { String prefix = getKey(directory); Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix)); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() // .withBucketName(this.bucketName) // .withPrefix(prefix) // .withDelimiter("/"); ObjectListing objectListing; objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); while (objectListing.isTruncated()) { objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); } return directoryContents; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e); } }
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider, @Nullable ClientConfiguration s3Config, String rootPath) { // Guess us-east-1. If wrong AWS will return a redirect with the correct endpoint AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config); if (rootPath.startsWith("/")) { rootPath = rootPath.substring(1); } if (!rootPath.endsWith("/")) { rootPath = rootPath + "/"; } try { // Any request will work but presumably the client has list access for stash so perform a list. s3.listObjects(new ListObjectsRequest() .withBucketName(bucket) .withPrefix(rootPath) .withDelimiter("/") .withMaxKeys(1)); // If this didn't error out then the presumed us-east-1 region was correct return "s3.us-east-1.amazonaws.com"; } catch (AmazonS3Exception e) { if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) { String endPoint = e.getAdditionalDetails().get("Endpoint"); // The end point is prefixed with the bucket name, so strip it return endPoint.substring(bucket.length() + 1); } throw e; } }
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider, @Nullable ClientConfiguration s3Config, String rootPath) { // Guess us-east-1. If wrong AWS will return a redirect with the correct endpoint AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config); if (rootPath.startsWith("/")) { rootPath = rootPath.substring(1); } if (!rootPath.endsWith("/")) { rootPath = rootPath + "/"; } try { // Any request will work but presumably the client has list access for stash so perform a list. s3.listObjects(new ListObjectsRequest() .withBucketName(bucket) .withPrefix(rootPath) .withDelimiter("/") .withMaxKeys(1)); // If this didn't error out then the presumed us-east-1 region was correct return "s3.us-east-1.amazonaws.com"; } catch (AmazonS3Exception e) { if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) { String endPoint = e.getAdditionalDetails().get("Endpoint"); // The end point is prefixed with the bucket name, so strip it return endPoint.substring(bucket.length() + 1); } throw e; } }
@Override protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException { List<String> directoryContents = new ArrayList<>(); try { String prefix = getKey(directory); Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix)); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() // .withBucketName(this.bucketName) // .withPrefix(prefix) // .withDelimiter("/"); ObjectListing objectListing; objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); while (objectListing.isTruncated()) { objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); } return directoryContents; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e); } }
@Override protected List<String> listDirectory(String directory) throws ResourceDoesNotExistException { List<String> directoryContents = new ArrayList<String>(); try { String prefix = getKey(directory); Pattern pattern = Pattern.compile(String.format(RESOURCE_FORMAT, prefix)); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() // .withBucketName(this.bucketName) // .withPrefix(prefix) // .withDelimiter("/"); ObjectListing objectListing; objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); while (objectListing.isTruncated()) { objectListing = this.amazonS3.listObjects(listObjectsRequest); directoryContents.addAll(getResourceNames(objectListing, pattern)); } return directoryContents; } catch (AmazonServiceException e) { throw new ResourceDoesNotExistException(String.format("'%s' does not exist", directory), e); } }
private Iterator<LocatedFileStatus> listPrefix(Path path) { String key = keyFromPath(path); if (!key.isEmpty()) { key += PATH_SEPARATOR; } ListObjectsRequest request = new ListObjectsRequest() .withBucketName(getBucketName(uri)) .withPrefix(key) .withDelimiter(PATH_SEPARATOR); STATS.newListObjectsCall(); Iterator<ObjectListing> listings = new AbstractSequentialIterator<ObjectListing>(s3.listObjects(request)) { @Override protected ObjectListing computeNext(ObjectListing previous) { if (!previous.isTruncated()) { return null; } return s3.listNextBatchOfObjects(previous); } }; return Iterators.concat(Iterators.transform(listings, this::statusFromListing)); }
public List<String> listS3Objects(final String bucketName, final String prefix) { final List<String> s3Objects = Lists.newArrayList(); try { logger.info("Listing objects"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/") .withBucketName(bucketName).withPrefix( prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { if (objectSummary.getKey().equals(prefix)) { continue; } s3Objects.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException e) { logger.error("Error Message: " + e.getMessage()); } return s3Objects; }
public List<String> listS3Objects(final String bucketName, final String prefix) { final List<String> s3Objects = Lists.newArrayList(); try { log.debug("Listing objects"); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withDelimiter("/") .withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { if (objectSummary.getKey().equals(prefix)) { continue; } s3Objects.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (final AmazonServiceException e) { log.error("Error Message: " + e.getMessage()); } return s3Objects; }
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) { final List<String> commonPrefixes = Lists.newArrayList(); try { logger.info("Listing objects"); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withDelimiter("/") .withBucketName(bucketName).withPrefix( prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); commonPrefixes.addAll(objectListing.getCommonPrefixes()); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException e) { logger.error("Error Message: " + e.getMessage()); } return commonPrefixes; }
@Override public ObjectListing listObjects(S3URI s3uri) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(s3uri.getBucket()).withDelimiter(S3URI.S3_DELIMITER);
/** * Searches for matching keys progressively. This means that instead of retrieving all keys given a prefix, it goes * down one level at a time and filters out all non-matching results. This avoids a lot of unused requests results. * WARNING: This method does not truncate results. Therefore all matching resources will be returned regardless of * the truncation. */ private void findProgressivelyWithPartialMatch(String bucketName, Set<Resource> resources, String prefix, String keyPattern) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter("/").withPrefix(prefix); ObjectListing objectListing = null; do { if (objectListing == null) { objectListing = this.amazonS3.listObjects(listObjectsRequest); } else { objectListing = this.amazonS3.listNextBatchOfObjects(objectListing); } Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries()); if (!newResources.isEmpty()) { resources.addAll(newResources); } for (String commonPrefix : objectListing.getCommonPrefixes()) { if (isKeyPathMatchesPartially(keyPattern, commonPrefix)) { findPathMatchingKeyInBucket(bucketName, resources, commonPrefix, keyPattern); } } } while (objectListing.isTruncated()); }
/** * Searches for matching keys progressively. This means that instead of retrieving all keys given a prefix, it goes * down one level at a time and filters out all non-matching results. This avoids a lot of unused requests results. * WARNING: This method does not truncate results. Therefore all matching resources will be returned regardless of * the truncation. */ private void findProgressivelyWithPartialMatch(String bucketName, Set<Resource> resources, String prefix, String keyPattern) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter("/").withPrefix(prefix); ObjectListing objectListing = null; do { if (objectListing == null) { objectListing = this.amazonS3.listObjects(listObjectsRequest); } else { objectListing = this.amazonS3.listNextBatchOfObjects(objectListing); } Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries()); if (!newResources.isEmpty()) { resources.addAll(newResources); } for (String commonPrefix : objectListing.getCommonPrefixes()) { if (isKeyPathMatchesPartially(keyPattern, commonPrefix)) { findPathMatchingKeyInBucket(bucketName, resources, commonPrefix, keyPattern); } } } while (objectListing.isTruncated()); }
@NotNull final String bucket, @NotNull final String key) { ObjectListing listing = client.listObjects( new ListObjectsRequest().withDelimiter("/").withPrefix(key) .withBucketName(bucket) );