private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> objects) { // NOTE: for encrypted objects, S3ObjectSummary.size() used below is NOT correct, // however, to get the correct size we'd need to make an additional request to get // user metadata, and in this case it doesn't matter. return objects.stream() .filter(object -> !object.getKey().endsWith(PATH_SEPARATOR)) .map(object -> new FileStatus( object.getSize(), false, 1, BLOCK_SIZE.toBytes(), object.getLastModified().getTime(), qualifiedPath(new Path(PATH_SEPARATOR + object.getKey())))) .map(this::createLocatedFileStatus) .iterator(); }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
public static Key loadKey(S3ObjectSummary obj) throws IOException { Key k = encodeKey(obj.getBucketName(), obj.getKey()); long size = obj.getSize(); Value val = new Value(k, (int) size, Value.S3); // Plain Value val.setdsk(); DKV.put(k, val); return k; }
&& !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER) ) { objectSummaries.add(s); totalSize += s.getSize(); } else { log.debug("Skipping download for object " + s.getKey()
public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail){ for( S3ObjectSummary obj : listing.getObjectSummaries() ) { try { Key k = PersistS3.loadKey(obj); JsonObject o = new JsonObject(); o.addProperty(KEY, k.toString()); o.addProperty(FILE, obj.getKey()); o.addProperty(VALUE_SIZE, obj.getSize()); succ.add(o); } catch( IOException e ) { JsonObject o = new JsonObject(); o.addProperty(FILE, obj.getKey()); o.addProperty(ERROR, e.getMessage()); fail.add(o); } } }
public static Key make(S3ObjectSummary obj, Futures fs) { String fname = obj.getKey(); Key k = Key.make("s3://" + obj.getBucketName() + "/" + fname); long size = obj.getSize(); Key k2 = Vec.newKey(k); new Frame(k).delete_and_lock(null); // Insert the top-level FileVec key into the store Vec v = new S3FileVec(k2,size); DKV.put(k2, v, fs); Frame fr = new Frame(k,new String[]{fname},new Vec[]{v}); fr.update(null); fr.unlock(null); return k; } private S3FileVec(Key key, long len) {super(key,len,Value.S3);}
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjectsV2(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
Map<Long, TemplateProp> s3ListVolume(S3TO s3) { String bucket = s3.getBucketName(); // List the objects in the source directory on S3 final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR); if (objectSummaries == null) { return null; } Map<Long, TemplateProp> tmpltInfos = new HashMap<Long, TemplateProp>(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); // String installPath = StringUtils.substringBeforeLast(key, // S3Utils.SEPARATOR); Long id = determineS3VolumeIdFromKey(key); // TODO: how to get volume template name TemplateProp tInfo = new TemplateProp(id.toString(), key, objectSummary.getSize(), objectSummary.getSize(), true, false); tmpltInfos.put(id, tInfo); } return tmpltInfos; }
Map<String, TemplateProp> s3ListTemplate(S3TO s3) { String bucket = s3.getBucketName(); // List the objects in the source directory on S3 final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR); if (objectSummaries == null) { return null; } Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); // String installPath = StringUtils.substringBeforeLast(key, // S3Utils.SEPARATOR); String uniqueName = determineS3TemplateNameFromKey(key); // TODO: isPublic value, where to get? TemplateProp tInfo = new TemplateProp(uniqueName, key, objectSummary.getSize(), objectSummary.getSize(), true, false); tmpltInfos.put(uniqueName, tInfo); } return tmpltInfos; }
s.getKey() + DEFAULT_DELIMITER)) { objectSummaries.add(s); totalSize += s.getSize(); } else { log.debug("Skipping download for object " + s.getKey()
@Override public SingularityS3LogMetadata call() throws Exception { Optional<Long> maybeStartTime = Optional.absent(); Optional<Long> maybeEndTime = Optional.absent(); if (!search.isExcludeMetadata()) { GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey()); Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata(); maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); } if (search.isListOnly()) { return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime); } else { GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt); String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString(); ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides(); downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER); downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER); GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt) .withResponseHeaders(downloadHeaders); String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString(); return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime); } }
@Override boolean objectChanged(ObjectMetadata metadata) { return summary.getSize() != metadata.getContentLength(); } }
/** * String information about a summary entry for debug messages. * @param summary summary object * @return string value */ public static String stringify(S3ObjectSummary summary) { StringBuilder builder = new StringBuilder(summary.getKey().length() + 100); builder.append(summary.getKey()).append(' '); builder.append("size=").append(summary.getSize()); return builder.toString(); }
private FileStatus createFileStatus(S3ObjectSummary objSummary, String hostName, Path path) throws IllegalArgumentException, IOException { String objKey = objSummary.getKey(); String newMergedPath = getMergedPath(hostName, path, objKey); return createFileStatus(objSummary.getSize(), objKey, objSummary.getLastModified(), new Path(newMergedPath)); }
private FileStatus createFileStatus(S3ObjectSummary objSummary, String hostName, Path path) throws IllegalArgumentException, IOException { String objKey = objSummary.getKey(); String newMergedPath = getMergedPath(hostName, path, objKey); return createFileStatus(objSummary.getSize(), objKey, objSummary.getLastModified(), new Path(newMergedPath)); }
@Override public DataRecord apply(S3ObjectSummary input) { return new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), input.getLastModified().getTime(), input.getSize()); } });
@Override public DataRecord apply(S3ObjectSummary input) { return new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), input.getLastModified().getTime(), input.getSize()); } });
public void updateBucketSummary(BucketSummary summary, List<S3ObjectSummary> summaries) { for (S3ObjectSummary element : summaries) { summary.setSize(summary.getSize() + element.getSize()); summary.setCount(summary.getCount() + 1); } }