Refine search
options.maxResults(maxKeys); PageSet<? extends StorageMetadata> set = blobStore.list(containerName, options); String nextMarker = set.getNextMarker(); if (nextMarker != null) { writeSimpleElement(xml, "IsTruncated", "true"); if (Quirks.OPAQUE_MARKERS.contains(blobStoreType)) { lastKeyToMarker.put(Maps.immutableEntry(containerName, Iterables.getLast(set).getName()), nextMarker); switch (metadata.getType()) { case FOLDER: commonPrefixes.add(metadata.getName()); continue; default:
blobStore.list(bucketName, new ListContainerOptions().recursive()); logger.trace(" Found {} files in bucket {}", pageSets.size(), bucketName);
protected Set<String> listContainers(BlobStore blobStore) { Set<String> containers = new LinkedHashSet<String>(); if (blobStore != null) { PageSet<? extends StorageMetadata> storageMetadatas = blobStore.list(); if (storageMetadatas != null && !storageMetadatas.isEmpty()) { for (StorageMetadata metadata : storageMetadatas) { containers.add(metadata.getName()); } } } return containers; }
private void advance() { if (marker != null) { options.afterMarker(marker); } PageSet<? extends StorageMetadata> set = blobStore.list( containerName, options); marker = set.getNextMarker(); iterator = set.iterator(); }
@Override public List<MultipartUpload> listMultipartUploads(String container) { ImmutableList.Builder<MultipartUpload> mpus = ImmutableList.builder(); ListContainerOptions options = new ListContainerOptions().prefix(MULTIPART_PREFIX).recursive(); int uuidLength = UUID.randomUUID().toString().length(); while (true) { PageSet<? extends StorageMetadata> pageSet = list(container, options); for (StorageMetadata sm : pageSet) { if (!sm.getName().endsWith("-stub")) { continue; } String uploadId = sm.getName().substring(MULTIPART_PREFIX.length(), MULTIPART_PREFIX.length() + uuidLength); String blobName = sm.getName().substring(MULTIPART_PREFIX.length() + uuidLength + 1); int index = blobName.lastIndexOf('-'); blobName = blobName.substring(0, index); mpus.add(MultipartUpload.create(container, blobName, uploadId, null, null)); } if (pageSet.isEmpty() || pageSet.getNextMarker() == null) { break; } options.afterMarker(pageSet.getNextMarker()); } return mpus.build(); }
public void execute(final String containerName, ListContainerOptions options) { String message = options.getDir() != null ? String.format("clearing path %s/%s", containerName, options.getDir()) : String.format("clearing container %s", containerName); options = options.clone(); if (options.isRecursive()) message += " recursively"; blobStore.list(containerName, options); for (StorageMetadata md : listing) { String fullPath = parentIsFolder(options, md) ? options.getDir() + "/" + md.getName() : md.getName(); switch (md.getType()) { case BLOB: break; String marker = listing.getNextMarker(); if (marker == null) { break;
public void testListBlobEndsWithDelimiter() { String containerName = "testListBlobEndsWithDelimiter"; blobStore.createContainerInLocation(null, containerName); blobStore.putBlob(containerName, blobStore.blobBuilder("foo/").payload(ByteSource.empty()).build()); PageSet<? extends StorageMetadata> results = blobStore.list(containerName, ListContainerOptions.Builder.prefix("foo/")); assertThat(results.size()).isEqualTo(1); assertThat(Iterables.get(results, 0).getName()).isEqualTo("foo/"); assertThat(Iterables.get(results, 0).getType()).isNotEqualTo(StorageType.RELATIVE_PATH); }
@Test(groups = { "integration", "live" }, dataProvider = "putTests") public void testPutObject(String name, String type, Object content, Object realObject) throws InterruptedException, IOException { PayloadBlobBuilder blobBuilder = view.getBlobStore().blobBuilder(name).payload(Payloads.newPayload(content)) .contentType(type); addContentMetadata(blobBuilder); if (content instanceof InputStream) { blobBuilder.calculateMD5(); } Blob blob = blobBuilder.build(); String container = getContainerName(); try { assertNotNull(view.getBlobStore().putBlob(container, blob)); blob = view.getBlobStore().getBlob(container, blob.getMetadata().getName()); validateMetadata(blob.getMetadata(), container, name); checkContentMetadata(blob); String returnedString = getContentAsStringOrNullAndClose(blob); assertEquals(returnedString, realObject); PageSet<? extends StorageMetadata> set = view.getBlobStore().list(container); assert set.size() == 1 : set; } finally { returnContainer(container); } }
try { BlobStore blobStore = blobStoreContext.getBlobStore(); ListContainerOptions listContainerOptions = new ListContainerOptions(); if (listRecursiveFlag == ListRecursiveFlag.TRUE) { listContainerOptions.recursive(); listContainerOptions.prefix(prefix); if (listRecursiveFlag == ListRecursiveFlag.FALSE) { listContainerOptions.delimiter(BucketName.BUCKET_SEPARATOR); LOGGER.debug("list container:" + objectHandle.getContainer() + " prefix:" + listContainerOptions.getPrefix() + " del:" + listContainerOptions.getDelimiter()); org.jclouds.blobstore.domain.PageSet<? extends org.jclouds.blobstore.domain.StorageMetadata> ps = blobStore.list(objectHandle.getContainer(), listContainerOptions); LinkedHashSet<StorageMetadata> set = new LinkedHashSet<StorageMetadata>(); for (org.jclouds.blobstore.domain.StorageMetadata s : ps) { StorageType type = s.getType()==null?null:StorageType.valueOf(s.getType().name()); Location location = copyLocation(new HashSet<String>(), s.getLocation()); StorageMetadata e = new StorageMetadataImpl(type, s.getProviderId(), s.getName(), location, s.getUri(), s.getETag(), s.getCreationDate(), s.getLastModified(), set.add(e); return new PageSetImpl<>(set, ps.getNextMarker());
/** Test that listing with a marker prefix matches the first key with that prefix. */ @Test public void testListMarkerPrefix() throws Exception { BlobStore blobStore = view.getBlobStore(); final String container = getContainerName(); try { blobStore.createContainerInLocation(null, container); blobStore.putBlob(container, blobStore.blobBuilder("a/a").payload("").build()); blobStore.putBlob(container, blobStore.blobBuilder("b/b").payload("").build()); ListContainerOptions options = new ListContainerOptions().afterMarker("b/").recursive(); PageSet<? extends StorageMetadata> res = blobStore.list(container, options); assertThat(res).hasSize(1); assertThat(res.iterator().next().getName()).isEqualTo("b/b"); } finally { returnContainer(container); } }
@SuppressWarnings("deprecation") @Override public Iterator<String> getNextObjectList(String prefix) throws IOException { List<String> al = new ArrayList<String>(); if (!iter.hasNext()) { if (ips.getNextMarker() == null) return al.iterator(); else { if (this.atmosStore) ips = blobStore.list(this.name, ListContainerOptions.Builder.recursive() .afterMarker(ips.getNextMarker()).inDirectory(prefix)); else if (this.b2Store) ips = blobStore.list(this.name, ListContainerOptions.Builder.recursive() .afterMarker(ips.getNextMarker()).prefix(prefix).maxResults(100)); else ips = blobStore.list(this.name, ListContainerOptions.Builder.recursive().afterMarker(ips.getNextMarker()).prefix(prefix)); // SDFSLogger.getLog().info("lsize=" + ips.size()); iter = ips.iterator(); } } while (iter.hasNext()) { String fn = iter.next().getName(); // SDFSLogger.getLog().info("fn=" + fn); if (!fn.endsWith(mdExt) && !fn.endsWith("/")) { al.add(fn); } } return al.iterator(); }
PageSet<? extends StorageMetadata> pageSet = view.getBlobStore().list(containerName); assertThat(pageSet).hasSize(3); assertThat(pageSet.getNextMarker()).isNull(); ListContainerOptions options = new ListContainerOptions().maxResults(1); pageSet = view.getBlobStore().list(containerName, options); assertThat(pageSet).hasSize(1); assertThat(pageSet.iterator().next().getName()).isEqualTo("asdf"); assertThat(pageSet.getNextMarker()).isNotNull(); options.afterMarker(pageSet.getNextMarker()); assertThat(pageSet.iterator().next().getName()).isEqualTo("boo/"); assertThat(pageSet.getNextMarker()).isNotNull(); options.afterMarker(pageSet.getNextMarker()); assertThat(pageSet.iterator().next().getName()).isEqualTo("cquux/"); assertThat(pageSet.getNextMarker()).isNull(); assertThat(pageSet.iterator().next().getName()).isEqualTo("boo/bar"); assertThat(pageSet.getNextMarker()).isNotNull(); options.afterMarker(pageSet.getNextMarker()); assertThat(pageSet.iterator().next().getName()).isEqualTo("boo/baz/"); assertThat(pageSet.getNextMarker()).isNull(); Iterator<? extends StorageMetadata> it = pageSet.iterator();
@Override public Iterable<? extends StorageMetadata> execute(String container, ListContainerOptions options) { try { boolean truncated = true; List<PageSet<? extends StorageMetadata>> listings = Lists.newArrayList(); while (truncated) { PageSet<? extends StorageMetadata> listing = connection.list(container, options); truncated = listing.getNextMarker() != null; if (truncated) { options = options instanceof ImmutableListContainerOptions ? options.clone() .afterMarker(listing.getNextMarker()) : options.afterMarker(listing .getNextMarker()); } listings.add(listing); } return Iterables.concat(listings); } catch (Exception e) { Throwables.propagateIfPossible(e, BlobRuntimeException.class); throw new BlobRuntimeException("Error getting resource metadata in container: " + container, e); } } }
@Test(dataProvider = "ignoreOnMacOSX") public void testListDirectoryBlobsS3FS() { blobStore.createContainerInLocation(null, CONTAINER_NAME); checkForContainerContent(CONTAINER_NAME, null); String d = TestUtils.createRandomBlobKey("directory-", ""); blobStore.putBlob(CONTAINER_NAME, createDirBlob(d + File.separator)); assertTrue(blobStore.blobExists(CONTAINER_NAME, d + File.separator)); ListContainerOptions options = ListContainerOptions.Builder .withDetails() .inDirectory(""); PageSet<? extends StorageMetadata> res = blobStore.list(CONTAINER_NAME, options); assertTrue(res.size() == 1); assertEquals(res.iterator().next().getName(), d + File.separator); }
public void testListMarkers() { String containerName = "testListMarkers"; blobStore.createContainerInLocation(null, containerName); blobStore.putBlob(containerName, blobStore.blobBuilder("abc").payload("").build()); blobStore.putBlob(containerName, blobStore.blobBuilder("foo/bar").payload("").build()); blobStore.putBlob(containerName, blobStore.blobBuilder("foo/baz").payload("").build()); blobStore.putBlob(containerName, blobStore.blobBuilder("qux").payload("").build()); PageSet<? extends StorageMetadata> results = blobStore.list( containerName, ListContainerOptions.Builder.maxResults(1)); assertThat(results.getNextMarker()).isEqualTo("abc"); results = blobStore.list(containerName, ListContainerOptions.Builder.maxResults(1).afterMarker(results.getNextMarker())); assertThat(results.getNextMarker()).isEqualTo("foo/"); results = blobStore.list(containerName, ListContainerOptions.Builder.maxResults(1).afterMarker(results.getNextMarker())); assertThat(results.getNextMarker()).isEqualTo(null); }
assert container.getNextMarker() == null; assert container.size() == 1 : container; assert container.getNextMarker() == null; assert container.size() == 0 : container; assert container.getNextMarker() == null; assert container.size() == 1 : container; assert container.getNextMarker() == null; assert container.size() == 10 : container; assert container.getNextMarker() == null; assert container.size() == 1 : container; container = view.getBlobStore().list(containerName, inDirectory(directory).recursive()); assert container.getNextMarker() == null; assert container.size() == 1 : container; assert container.getNextMarker() == null; assert container.size() == 0 : container; assert container.getNextMarker() == null; assert container.size() == 1 : container; assert container.getNextMarker() == null; assert container.size() == 0 : container; } finally { returnContainer(containerName);
ListContainerOptions opt = new ListContainerOptions(); PageSet<? extends StorageMetadata> pageSet = blobStore.list(container, opt); LOGGER.debug("CELLAR CLOUD: storage contains a page set of size {}", pageSet.size()); for (StorageMetadata md : pageSet) { if (md.getType() != StorageType.BLOB) { String ip = md.getName(); Object obj = readBlob(container, ip); if (obj == null) { } else { LOGGER.debug("CELLAR CLOUD: remove container {}", ip); blobStore.removeBlob(container, ip); } else { LOGGER.debug("CELLAR CLOUD: remove container {}", ip); blobStore.removeBlob(container, ip);
@Test(groups = { "integration", "live" }) public void testListContainerMarker() throws InterruptedException { String containerName = getContainerName(); try { addAlphabetUnderRoot(containerName); PageSet<? extends StorageMetadata> container = view.getBlobStore().list(containerName, maxResults(1)); assert container.getNextMarker() != null; assertEquals(container.size(), 1); String marker = container.getNextMarker(); container = view.getBlobStore().list(containerName, afterMarker(marker)); assertEquals(container.getNextMarker(), null); assert container.size() == 25 : String.format("size should have been 25, but was %d: %s", container.size(), container); assert container.getNextMarker() == null; } finally { returnContainer(containerName); } }
opts.afterMarker(marker); PageSet<? extends StorageMetadata> pageSet = store.list(container, opts); log.debug("Loaded blobs list [size={}, marker={}]", pageSet.size(), marker); if (metaData.getType() == StorageType.BLOB) { String name = metaData.getName(); marker = pageSet.getNextMarker(); } while (marker != null);
@Override public boolean hasNext() { if ((set == null) || (queue == null)) { set = store.list(cloudContainer, maxResults(BATCH)); loadElements(set); } if (!queue.isEmpty()) { return true; } else if (set.getNextMarker() != null) { set = store.list(cloudContainer, maxResults(BATCH).afterMarker(set.getNextMarker())); loadElements(set); if (!queue.isEmpty()) { return true; } } return false; }