public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = s3Service.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new S3ObjectsChunk( prefix, delimiter, (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException se) { result = new S3ServiceException(se); } }
public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = s3Service.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new S3ObjectsChunk( prefix, delimiter, (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException se) { result = new S3ServiceException(se); } }
@Override public void iterationInit(boolean deep) { try { bs3Service = new RestS3Service(awsCredentials); StorageObjectsChunk ck = bs3Service.listObjectsChunked( this.getName(), null, null, 1000, null); this.compressedLength.set(0); this.currentLength.set(0); obj = ck.getObjects(); this.lastKey = null; objPos = 0; } catch (ServiceException e) { SDFSLogger.getLog().error("unable to initialize", e); } }
public List<ListItem> listAll(String path) { m_logger.debug("Start list all: " + path); try { List<ListItem> result = new ArrayList<>(); String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); for(int i = 0; i < objects.length; i++) { String key = objects[i].getKey(); if(key.endsWith("/")) key = key.substring(0, key.length() - 1); key = key.substring(path.length(), key.length()); ListItem item = new ListItem(key, objects[i].getContentLength() != 0); result.add(item); } if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } return result; } catch (ServiceException e) { throw new RuntimeException(e); } }
public static void listBucketAWS(String bucketName, String awsAccessKey, String awsSecretKey) { try { System.out .println("Listing Objects in Bucket [" + bucketName + "]"); AWSCredentials bawsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); S3Service bs3Service = new RestS3Service(bawsCredentials); StorageObjectsChunk ck = bs3Service.listObjectsChunked(bucketName, null, null, 100, null); StorageObject[] obj = ck.getObjects(); String lastKey = null; while (obj.length > 0) { for (int i = 0; i < obj.length; i++) { lastKey = obj[i].getKey(); System.out.println(lastKey); } ck = bs3Service.listObjectsChunked(bucketName, null, null, 100, lastKey); obj = ck.getObjects(); } } catch (ServiceException e) { e.printStackTrace(); } }
StorageObjectsChunk initialChunk = restService.listObjectsChunked( bucketName, null, delimiter, 1000, null, true);
StorageObjectsChunk initialChunk = restService.listObjectsChunked( bucketName, null, delimiter, 1000, null, true);
public void deleteAll(String path) { try { String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects to delete: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); if(objects.length == 0) break; String[] names = new String[objects.length]; for(int i = 0; i < objects.length; i++) { names[i] = objects[i].getKey(); } m_s3service.deleteMultipleObjects(BUCKET, names); m_logger.trace("DeleteObjects: {}", objects.length); // do not inc() because delete requests are not counted if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } } catch (ServiceException e) { throw new RuntimeException(e); } }
String priorLastKey = null; do { StorageObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked( currentSelectedBucket.getName(), prefix, delimiter, BUCKET_LIST_CHUNKING_SIZE, priorLastKey);
String priorLastKey = null; do { StorageObjectsChunk chunk = s3ServiceMulti.getS3Service().listObjectsChunked( currentSelectedBucket.getName(), prefix, delimiter, BUCKET_LIST_CHUNKING_SIZE, priorLastKey);
try { if (objPos >= obj.length) { StorageObjectsChunk ck = bs3Service.listObjectsChunked( this.getName(), null, null, 1000, lastKey); obj = ck.getObjects();
@Override public void delete() throws IOException { // Note: DELETE on a non-existing resource is a successful request, so we need this check if(!exists()) throw new IOException(); try { // Make sure that the directory is empty, abort if not. // Note that we must not count the parent directory (this file). boolean isDirectory = isDirectory(); if(isDirectory && service.listObjectsChunked(bucketName, getObjectKey(true), "/", 2, null, false).getObjects().length>1) throw new IOException("Directory not empty"); service.deleteObject(bucketName, getObjectKey(isDirectory)); // Update file attributes locally atts.setExists(false); atts.setDirectory(false); atts.setSize(0); } catch(S3ServiceException e) { throw getIOException(e); } }
protected AbstractFile[] listObjects(String bucketName, String prefix, S3File parent) throws IOException { try { S3ObjectsChunk chunk = service.listObjectsChunked(bucketName, prefix, "/", Constants.DEFAULT_OBJECT_LIST_CHUNK_SIZE, null, true); org.jets3t.service.model.S3Object objects[] = chunk.getObjects(); String[] commonPrefixes = chunk.getCommonPrefixes();
prefix += PATH_DELIMITER; StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), prefix, delimiter, maxListingLength, priorLastKey);
private PartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey) throws IOException { try { if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), prefix, delimiter, maxListingLength, priorLastKey); FileMetadata[] fileMetadata = new FileMetadata[chunk.getObjects().length]; for (int i = 0; i < fileMetadata.length; i++) { S3Object object = chunk.getObjects()[i]; fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime()); } return new PartialListing(chunk.getPriorLastKey(), fileMetadata, chunk.getCommonPrefixes()); } catch (S3ServiceException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } throw new S3Exception(e); } }
private PartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey) throws IOException { try { if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), prefix, delimiter, maxListingLength, priorLastKey); FileMetadata[] fileMetadata = new FileMetadata[chunk.getObjects().length]; for (int i = 0; i < fileMetadata.length; i++) { S3Object object = chunk.getObjects()[i]; fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime()); } return new PartialListing(chunk.getPriorLastKey(), fileMetadata, chunk.getCommonPrefixes()); } catch (S3ServiceException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } throw new S3Exception(e); } }