@Override public long getLastModified() { return objectSummary.getLastModified().getTime(); }
private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> objects) { // NOTE: for encrypted objects, S3ObjectSummary.size() used below is NOT correct, // however, to get the correct size we'd need to make an additional request to get // user metadata, and in this case it doesn't matter. return objects.stream() .filter(object -> !object.getKey().endsWith(PATH_SEPARATOR)) .map(object -> new FileStatus( object.getSize(), false, 1, BLOCK_SIZE.toBytes(), object.getLastModified().getTime(), qualifiedPath(new Path(PATH_SEPARATOR + object.getKey())))) .map(this::createLocatedFileStatus) .iterator(); }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
continue; final long latestModified = objectSummary.getLastModified().getTime(); if (latestModified >= mostRecent) { mostRecent = latestModified;
/** * Returns the "version" (aka last modified timestamp) of the URI * * @param uri The URI to check the last timestamp * * @return The time in ms of the last modification of the URI in String format * * @throws IOException */ @Override public String getVersion(URI uri) throws IOException { try { final S3Coords coords = new S3Coords(checkURI(uri)); final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path); return StringUtils.format("%d", objectSummary.getLastModified().getTime()); } catch (AmazonServiceException e) { if (S3Utils.isServiceExceptionRecoverable(e)) { // The recoverable logic is always true for IOException, so we want to only pass IOException if it is recoverable throw new IOE(e, "Could not fetch last modified timestamp from URI [%s]", uri); } else { throw new RE(e, "Error fetching last modified timestamp from URI [%s]", uri); } } }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjectsV2(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }
int indexOfLastSlash = itemName.lastIndexOf("/"); Map<String, Object> objectMetadataMap = new HashMap<>(); long updatedTimestamp = summary.getLastModified().getTime();
@Override public SingularityS3LogMetadata call() throws Exception { Optional<Long> maybeStartTime = Optional.absent(); Optional<Long> maybeEndTime = Optional.absent(); if (!search.isExcludeMetadata()) { GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey()); Map<String, String> objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata(); maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); } if (search.isListOnly()) { return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime); } else { GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt); String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString(); ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides(); downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER); downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER); GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) .withMethod(HttpMethod.GET) .withExpiration(expireAt) .withResponseHeaders(downloadHeaders); String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString(); return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime); } }
@Override public boolean test(S3ObjectSummary summary) { long lastModified = summary.getLastModified().getTime(); boolean applies = timeStamp >= lastModified; return applies; } }
@Override public boolean test(S3ObjectSummary summary) { long lastModified = summary.getLastModified().getTime(); boolean applies = timeStamp >= lastModified; return applies; } }
@Override protected long getModified(S3ObjectSummary file) { return file.getLastModified().getTime(); }
@Override protected long getModified(S3ObjectSummary file) { return file.getLastModified().getTime(); }
private boolean matchBillingRegexAndWasNotProcessed(S3ObjectSummary o) { return reportPattern.matcher(o.getKey()).matches() && !moduleData.wasProcessed(o.getKey(), o.getLastModified(), extractDatePrefix(reportPattern, o)); }
private FileStatus createFileStatus(S3ObjectSummary objSummary, String hostName, Path path) throws IllegalArgumentException, IOException { String objKey = objSummary.getKey(); String newMergedPath = getMergedPath(hostName, path, objKey); return createFileStatus(objSummary.getSize(), objKey, objSummary.getLastModified(), new Path(newMergedPath)); }
private FileStatus createFileStatus(S3ObjectSummary objSummary, String hostName, Path path) throws IllegalArgumentException, IOException { String objKey = objSummary.getKey(); String newMergedPath = getMergedPath(hostName, path, objKey); return createFileStatus(objSummary.getSize(), objKey, objSummary.getLastModified(), new Path(newMergedPath)); }
@Override public DataRecord apply(S3ObjectSummary input) { return new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), input.getLastModified().getTime(), input.getSize()); } });
@Override public DataRecord apply(S3ObjectSummary input) { return new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), input.getLastModified().getTime(), input.getSize()); } });
public static ObjectSummary copyOf(S3ObjectSummary mutable) { checkNotNull(mutable, "mutable"); Builder builder = builder(); builder.withBucket(mutable.getBucketName()); builder.withKey(mutable.getKey()); builder.withEtag(mutable.getETag()); builder.withSize(mutable.getSize()); builder.withLastModified(mutable.getLastModified().getTime()); builder.withOwner(Owner.copyOf(mutable.getOwner())); builder.withStorageClass(mutable.getStorageClass()); return builder.build(); }