@Override public ObjectListingChunk getNextChunk() throws IOException { if (!mChunk.isListingComplete()) { StorageObjectsChunk nextChunk = getObjectListingChunk(mChunk.getPrefix(), mChunk.getDelimiter(), mChunk.getPriorLastKey()); if (nextChunk != null) { return new GCSObjectListingChunk(nextChunk); } } return null; } }
public List<ListItem> listAll(String path) { m_logger.debug("Start list all: " + path); try { List<ListItem> result = new ArrayList<>(); String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); for(int i = 0; i < objects.length; i++) { String key = objects[i].getKey(); if(key.endsWith("/")) key = key.substring(0, key.length() - 1); key = key.substring(path.length(), key.length()); ListItem item = new ListItem(key, objects[i].getContentLength() != 0); result.add(item); } if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } return result; } catch (ServiceException e) { throw new RuntimeException(e); } }
@Override public ObjectListingChunk getNextChunk() throws IOException { if (!mChunk.isListingComplete()) { StorageObjectsChunk nextChunk = getObjectListingChunk(mChunk.getPrefix(), mChunk.getDelimiter(), mChunk.getPriorLastKey()); if (nextChunk != null) { return new GCSObjectListingChunk(nextChunk); } } return null; } }
public void deleteAll(String path) { try { String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects to delete: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); if(objects.length == 0) break; String[] names = new String[objects.length]; for(int i = 0; i < objects.length; i++) { names[i] = objects[i].getKey(); } m_s3service.deleteMultipleObjects(BUCKET, names); m_logger.trace("DeleteObjects: {}", objects.length); // do not inc() because delete requests are not counted if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } } catch (ServiceException e) { throw new RuntimeException(e); } }
@Override public boolean hasNext() { if (objectsChunk == null) { objectsChunk = listObjectsChunkedAfter(""); objectsChunkOffset = 0; } if (objectsChunk.getObjects().length <= objectsChunkOffset) { if (objectsChunk.isListingComplete()) { return false; } else { objectsChunk = listObjectsChunkedAfter(objectsChunk.getPriorLastKey()); objectsChunkOffset = 0; } } return true; }
.forEach(storageObject -> objects.add((S3Object) storageObject)); lastKey = objectsChunk.getPriorLastKey(); } while (!objectsChunk.isListingComplete());