@Override public String apply(S3ObjectSummary summary) { return summary.getKey(); } }
@Override public boolean apply(S3ObjectSummary summary) { return fromKey(summary.getKey()) != null; } }
@Override public BackupMetaData apply(S3ObjectSummary summary) { return fromKey(summary.getKey()); } }
void deleteObjectsInFolder(String bucketName, String folderPath) { for (S3ObjectSummary file : s3.listObjects(bucketName, folderPath).getObjectSummaries()){ s3.deleteObject(bucketName, file.getKey()); } }
/** * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. * * <pre> * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}} * </pre> */ private static URI toUri(S3ObjectSummary object) { final String originalAuthority = object.getBucketName(); final String originalPath = object.getKey(); final String authority = originalAuthority.endsWith("/") ? originalAuthority.substring(0, originalAuthority.length() - 1) : originalAuthority; final String path = originalPath.startsWith("/") ? originalPath.substring(1) : originalPath; return URI.create(StringUtils.format("s3://%s/%s", authority, path)); } }
private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> objects) { // NOTE: for encrypted objects, S3ObjectSummary.size() used below is NOT correct, // however, to get the correct size we'd need to make an additional request to get // user metadata, and in this case it doesn't matter. return objects.stream() .filter(object -> !object.getKey().endsWith(PATH_SEPARATOR)) .map(object -> new FileStatus( object.getSize(), false, 1, BLOCK_SIZE.toBytes(), object.getLastModified().getTime(), qualifiedPath(new Path(PATH_SEPARATOR + object.getKey())))) .map(this::createLocatedFileStatus) .iterator(); }
private Iterator<String> createIterator() { if (objectListing == null) initListing(); List<String> temp = Lists.newArrayList(); for (S3ObjectSummary summary : objectListing.getObjectSummaries()) { temp.add(summary.getKey()); } return temp.iterator(); }
for (S3ObjectSummary objectSummary : current.getObjectSummaries()) { S3Object object = s3.getObject(new GetObjectRequest(bucketName, objectSummary.getKey())); System.out.println(object.getKey()); object.close(); }
/** * Returns the objects names in a bucket. * * @param bucketName * the bucket whose contents will be listed. * @return the set of names of objects contained in a bucket. * @throws Exception */ public String[] getS3ObjectsNames( String bucketName ) throws Exception { Bucket bucket = getBucket( bucketName ); if ( bucket == null ) { throw new Exception( Messages.getString( "S3DefaultService.Exception.UnableToFindBucket.Message", bucketName ) ); } return getS3Objects( bucket ).getObjectSummaries().stream().map( b -> b.getKey() ).toArray( String[]::new ); }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
@Override public ObjectStatus[] getObjectStatuses() { List<S3ObjectSummary> objects = mResult.getObjectSummaries(); ObjectStatus[] ret = new ObjectStatus[objects.size()]; int i = 0; for (S3ObjectSummary obj : objects) { ret[i++] = new ObjectStatus(obj.getKey(), obj.getETag(), obj.getSize(), obj.getLastModified().getTime()); } return ret; }
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
/** * Returns an input stream for a s3 object. The returned input stream is not thread-safe. */ @Override public InputStream openInputStream() throws IOException { try { if (s3Object == null) { // lazily promote to full GET s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); } final InputStream in = s3Object.getObjectContent(); final Closer closer = Closer.create(); closer.register(in); closer.register(s3Object); return new FilterInputStream(in) { @Override public void close() throws IOException { closer.close(); } }; } catch (AmazonServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } }
/** * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket. * * @param s3Client s3 client * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request() .withBucketName(bucket) .withPrefix(key) .withMaxKeys(1); final ListObjectsV2Result result = s3Client.listObjectsV2(request); if (result.getKeyCount() == 0) { throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); } final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); } return objectSummary; } }
public static Key loadKey(S3ObjectSummary obj) throws IOException { Key k = encodeKey(obj.getBucketName(), obj.getKey()); long size = obj.getSize(); Value val = new Value(k, (int) size, Value.S3); // Plain Value val.setdsk(); DKV.put(k, val); return k; }
public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail){ for( S3ObjectSummary obj : listing.getObjectSummaries() ) { try { Key k = PersistS3.loadKey(obj); JsonObject o = new JsonObject(); o.addProperty(KEY, k.toString()); o.addProperty(FILE, obj.getKey()); o.addProperty(VALUE_SIZE, obj.getSize()); succ.add(o); } catch( IOException e ) { JsonObject o = new JsonObject(); o.addProperty(FILE, obj.getKey()); o.addProperty(ERROR, e.getMessage()); fail.add(o); } } }
public static Key make(S3ObjectSummary obj, Futures fs) { String fname = obj.getKey(); Key k = Key.make("s3://" + obj.getBucketName() + "/" + fname); long size = obj.getSize(); Key k2 = Vec.newKey(k); new Frame(k).delete_and_lock(null); // Insert the top-level FileVec key into the store Vec v = new S3FileVec(k2,size); DKV.put(k2, v, fs); Frame fr = new Frame(k,new String[]{fname},new Vec[]{v}); fr.update(null); fr.unlock(null); return k; } private S3FileVec(Key key, long len) {super(key,len,Value.S3);}
@Override public VersionListing listVersions() { VersionListing versionListing = new VersionListing(); this.objectListing = client.listObjects(listObjectsRequest); for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { S3VersionSummary versionSummary = new S3VersionSummary(); versionSummary.setBucketName(objectSummary.getBucketName()); versionSummary.setETag(objectSummary.getETag()); versionSummary.setKey(objectSummary.getKey()); versionSummary.setLastModified(objectSummary.getLastModified()); versionSummary.setOwner(objectSummary.getOwner()); versionSummary.setSize(objectSummary.getSize()); versionSummary.setStorageClass(objectSummary.getStorageClass()); versionSummary.setIsLatest(true); versionListing.getVersionSummaries().add(versionSummary); } return versionListing; }