/** * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. * * <pre> * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}} * </pre> */ private static URI toUri(S3ObjectSummary object) { final String originalAuthority = object.getBucketName(); final String originalPath = object.getKey(); final String authority = originalAuthority.endsWith("/") ? originalAuthority.substring(0, originalAuthority.length() - 1) : originalAuthority; final String path = originalPath.startsWith("/") ? originalPath.substring(1) : originalPath; return URI.create(StringUtils.format("s3://%s/%s", authority, path)); } }
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
mostRecent = latestModified; latest = new URI( StringUtils.format("s3://%s/%s", objectSummary.getBucketName(), objectSummary.getKey()) );
/** * Returns an input stream for a s3 object. The returned input stream is not thread-safe. */ @Override public InputStream openInputStream() throws IOException { try { if (s3Object == null) { // lazily promote to full GET s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); } final InputStream in = s3Object.getObjectContent(); final Closer closer = Closer.create(); closer.register(in); closer.register(s3Object); return new FilterInputStream(in) { @Override public void close() throws IOException { closer.close(); } }; } catch (AmazonServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } }
/** * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket. * * @param s3Client s3 client * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request() .withBucketName(bucket) .withPrefix(key) .withMaxKeys(1); final ListObjectsV2Result result = s3Client.listObjectsV2(request); if (result.getKeyCount() == 0) { throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); } final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); } return objectSummary; } }
public static Key loadKey(S3ObjectSummary obj) throws IOException { Key k = encodeKey(obj.getBucketName(), obj.getKey()); long size = obj.getSize(); Value val = new Value(k, (int) size, Value.S3); // Plain Value val.setdsk(); DKV.put(k, val); return k; }
GetObjectRequest req = new GetObjectRequest(summary.getBucketName(), summary.getKey()) .<GetObjectRequest>withGeneralProgressListener( listener);
public static Key make(S3ObjectSummary obj, Futures fs) { String fname = obj.getKey(); Key k = Key.make("s3://" + obj.getBucketName() + "/" + fname); long size = obj.getSize(); Key k2 = Vec.newKey(k); new Frame(k).delete_and_lock(null); // Insert the top-level FileVec key into the store Vec v = new S3FileVec(k2,size); DKV.put(k2, v, fs); Frame fr = new Frame(k,new String[]{fname},new Vec[]{v}); fr.update(null); fr.unlock(null); return k; } private S3FileVec(Key key, long len) {super(key,len,Value.S3);}
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjectsV2(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client();
@Override public SingularityS3LogMetadata call() throws Exception { Optional<Long> maybeStartTime = Optional.absent(); Optional<Long> maybeEndTime = Optional.absent(); if (!search.isExcludeMetadata()) { GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey()); Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata(); maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); } if (search.isListOnly()) { return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime); } else { GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt); String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString(); ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides(); downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER); downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER); GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt) .withResponseHeaders(downloadHeaders); String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString(); return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime); } }
new GetObjectRequest(summary.getBucketName(), summary.getKey()) .withGeneralProgressListener(
@Override public int compareTo(S3ObjectSummary o) { if (getBucketName().compareTo(o.getBucketName()) == 0) { return getKey().compareTo(o.getKey()); } else { return getBucketName().compareTo(o.getBucketName()); } }
/** * Creates a new S3 writable configuration. * @param s3 An S3 {@link AmazonS3 client}. * @param s3ObjectSummary The S3 object {@link S3ObjectSummary summary}. * @param version The version of the configuration. */ public S3WritableConfiguration(AmazonS3 s3, S3ObjectSummary s3ObjectSummary, String version) { this.s3 = s3; this.s3Object = s3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); this.version = version; }
/** * Creates a new S3 writable configuration. * @param s3 An S3 {@link AmazonS3 client}. * @param s3ObjectSummary The S3 object {@link S3ObjectSummary summary}. * @param version The version of the configuration. */ public S3WritableConfiguration(AmazonS3 s3, S3ObjectSummary s3ObjectSummary, String version) { this.s3 = s3; this.s3Object = s3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); this.version = version; }
public static Key loadKey(ObjectListing listing, S3ObjectSummary obj) throws IOException { // Note: Some of S3 implementations does not fill bucketName of returned object (for example, Minio). // So guess it based on returned ObjectListing String bucketName = obj.getBucketName() == null ? listing.getBucketName() : obj.getBucketName(); return S3FileVec.make(encodePath(bucketName, obj.getKey()),obj.getSize()); }
/** * Get S3 object using object summary * * @param objectSummary {@link S3ObjectSummary} for key to retrieve * @return {@link S3Object} read from S3 */ public S3Object getObject(S3ObjectSummary objectSummary) { getObjectRequest = new GetObjectRequest(objectSummary.getBucketName(), objectSummary.getKey()); return s3Client.getObject(getObjectRequest); }
@RequestMapping(method = RequestMethod.GET) public List<Resource<S3ObjectSummary>> getBucketResources() { ObjectListing objectListing = amazonS3Template.getAmazonS3Client() .listObjects(new ListObjectsRequest() .withBucketName(bucketName)); return objectListing.getObjectSummaries() .stream() .map(a -> new Resource<>(a, new Link(String.format("https://s3.amazonaws.com/%s/%s", a.getBucketName(), a.getKey())).withRel("url"))) .collect(Collectors.toList()); }
public static ObjectSummary copyOf(S3ObjectSummary mutable) { checkNotNull(mutable, "mutable"); Builder builder = builder(); builder.withBucket(mutable.getBucketName()); builder.withKey(mutable.getKey()); builder.withEtag(mutable.getETag()); builder.withSize(mutable.getSize()); builder.withLastModified(mutable.getLastModified().getTime()); builder.withOwner(Owner.copyOf(mutable.getOwner())); builder.withStorageClass(mutable.getStorageClass()); return builder.build(); }