@Override @Nullable protected ObjectListingChunk getObjectListingChunk(String key, boolean recursive) throws IOException { String delimiter = recursive ? "" : PATH_SEPARATOR; key = PathUtils.normalizePath(key, PATH_SEPARATOR); // In case key is root (empty string) do not normalize prefix. key = key.equals(PATH_SEPARATOR) ? "" : key; if (mConf.isSet(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1) && mConf .get(PropertyKey.UNDERFS_S3A_LIST_OBJECTS_VERSION_1).equals(Boolean.toString(true))) { ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ObjectListing result = getObjectListingChunkV1(request); if (result != null) { return new S3AObjectListingChunkV1(request, result); } } else { ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(mBucketName).withPrefix(key) .withDelimiter(delimiter).withMaxKeys(getListingChunkLength(mAlluxioConf)); ListObjectsV2Result result = getObjectListingChunk(request); if (result != null) { return new S3AObjectListingChunk(request, result); } } return null; }
private boolean fileExistsinS3(String bucketName, String path) { return s3Client.listObjects(new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(path.substring(1)) .withMaxKeys(1)).getObjectSummaries().isEmpty(); }
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider, @Nullable ClientConfiguration s3Config, String rootPath) { // Guess us-east-1. If wrong AWS will return a redirect with the correct endpoint AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config); if (rootPath.startsWith("/")) { rootPath = rootPath.substring(1); } if (!rootPath.endsWith("/")) { rootPath = rootPath + "/"; } try { // Any request will work but presumably the client has list access for stash so perform a list. s3.listObjects(new ListObjectsRequest() .withBucketName(bucket) .withPrefix(rootPath) .withDelimiter("/") .withMaxKeys(1)); // If this didn't error out then the presumed us-east-1 region was correct return "s3.us-east-1.amazonaws.com"; } catch (AmazonS3Exception e) { if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) { String endPoint = e.getAdditionalDetails().get("Endpoint"); // The end point is prefixed with the bucket name, so strip it return endPoint.substring(bucket.length() + 1); } throw e; } }
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider, @Nullable ClientConfiguration s3Config, String rootPath) { // Guess us-east-1. If wrong AWS will return a redirect with the correct endpoint AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config); if (rootPath.startsWith("/")) { rootPath = rootPath.substring(1); } if (!rootPath.endsWith("/")) { rootPath = rootPath + "/"; } try { // Any request will work but presumably the client has list access for stash so perform a list. s3.listObjects(new ListObjectsRequest() .withBucketName(bucket) .withPrefix(rootPath) .withDelimiter("/") .withMaxKeys(1)); // If this didn't error out then the presumed us-east-1 region was correct return "s3.us-east-1.amazonaws.com"; } catch (AmazonS3Exception e) { if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) { String endPoint = e.getAdditionalDetails().get("Endpoint"); // The end point is prefixed with the bucket name, so strip it return endPoint.substring(bucket.length() + 1); } throw e; } }
private Iterator<S3ObjectSummary> getS3ObjectSummaries(final String prefix) { final int prefixLength = prefix.length(); Iterator<S3ObjectSummary> allSummaries = Iterators.concat(new AbstractIterator<Iterator<S3ObjectSummary>>() { String marker = null; ObjectListing response; protected Iterator<S3ObjectSummary> computeNext() { if (response == null || response.isTruncated()) { response = _s3.listObjects(new ListObjectsRequest() .withBucketName(_bucket) .withPrefix(prefix) .withDelimiter("/") .withMarker(marker) .withMaxKeys(1000)); marker = response.getNextMarker(); return response.getObjectSummaries().iterator(); } return endOfData(); } }); // Sometimes the prefix itself can come back as a result. Filter that entry out. return Iterators.filter(allSummaries, new Predicate<S3ObjectSummary>() { @Override public boolean apply(S3ObjectSummary summary) { return summary.getKey().length() > prefixLength; } }); } }
private Iterator<S3ObjectSummary> getS3ObjectSummaries(final String prefix) { final int prefixLength = prefix.length(); Iterator<S3ObjectSummary> allSummaries = Iterators.concat(new AbstractIterator<Iterator<S3ObjectSummary>>() { String marker = null; ObjectListing response; protected Iterator<S3ObjectSummary> computeNext() { if (response == null || response.isTruncated()) { response = _s3.listObjects(new ListObjectsRequest() .withBucketName(_bucket) .withPrefix(prefix) .withDelimiter("/") .withMarker(marker) .withMaxKeys(1000)); marker = response.getNextMarker(); return response.getObjectSummaries().iterator(); } return endOfData(); } }); // Sometimes the prefix itself can come back as a result. Filter that entry out. return Iterators.filter(allSummaries, new Predicate<S3ObjectSummary>() { @Override public boolean apply(S3ObjectSummary summary) { return summary.getKey().length() > prefixLength; } }); } }
.withPrefix(prefix) .withMarker(_marker) .withMaxKeys(1000));
.withPrefix(prefix) .withMarker(_marker) .withMaxKeys(1000));
.withBucketName(this.s3ReaderConfiguration.getBucket()) .withPrefix(s3ReaderConfiguration.getReaderPath()) .withMaxKeys(500);