blobStore.list(bucketName, new ListContainerOptions().recursive());
final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();
@Override protected void after() { logger.debug("S3 proxy is stopping"); try { s3Proxy.stop(); BlobStore blobStore = blobStoreContext.getBlobStore(); for (StorageMetadata metadata : blobStore.list()) { blobStore.deleteContainer(metadata.getName()); } blobStoreContext.close(); } catch (Exception e) { throw new RuntimeException("Unable to stop S3 proxy", e); } FileUtils.deleteQuietly(blobStoreLocation); logger.debug("S3 proxy has stopped"); }
private void handleContainerList(HttpServletResponse response, BlobStore blobStore) throws IOException { PageSet<? extends StorageMetadata> buckets = blobStore.list();
public synchronized void downloadNewFiles(File downloadDir) { log.info("Downloading new files since {}", lastMarker); BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/ BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); //Gets key within the time range specified NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages); //Download only for keys within that range for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) { log.info("Downloading file: " + blobMeta.getValue()); downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue())); lastMarker = blobMeta.getValue(); synchronized (CloudFilesManager.this) { // this is where we resume from. MarkerUtils.writeLastMarker(blobMeta.getValue()); } } log.info("Updated the last marker value as " + lastMarker); }
public synchronized boolean hasNewFiles() { // see if there are any files since lastMarker. BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker); boolean emptiness = getBlobsWithinRange(pages).isEmpty(); if(emptiness) { log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME)); } else { log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME)); } return !emptiness; }
PageSet<? extends StorageMetadata> set = blobStore.list(containerName, options);
@Override protected void doCall(BlobStore blobStore, String indent) throws Exception { for (String containerName : names) { Set<? extends StorageMetadata> contents = blobStore.list(containerName); stdout.println(indent+"Container "+containerName+" {"); for (StorageMetadata content : contents) { stdout.println(indent+"\t"+content); } stdout.println(indent+"}"); } } }
@Override public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) { return delegate().list(container, options); }
@Override public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) { return delegate().list(container, options); }
@Override public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) { return delegate().list(container, options); }
protected Set<String> listBlobs(BlobStore blobStore, String container) { Set<String> blobs = new LinkedHashSet<String>(); if (blobStore != null && blobStore.containerExists(container)) { PageSet<? extends StorageMetadata> storageMetadatas = blobStore.list(container); if (storageMetadatas != null && !storageMetadatas.isEmpty()) { for (StorageMetadata metadata : storageMetadatas) { blobs.add(metadata.getName()); } } } return blobs; }
public static void main(String[] args) throws Exception { BatchJCloudChunkStore st = new BatchJCloudChunkStore(); st.name = "datishpool0"; Properties p = new Properties(); p.setProperty("service-type", "b2"); st.checkAccess("a6ba900eebf7", "001c60230bbe493db4bbb8d5031a9707628790c7a9", p); System.out.println("zzz"); PageSet<? extends StorageMetadata> zips; zips = st.blobStore.list(st.name, ListContainerOptions.Builder.prefix("files/").maxResults(100)); System.out.println(zips.size()); }
private void assertContainerEmptyDeleting(String container, String name) { Iterable<? extends StorageMetadata> listing = Iterables.filter(view.getBlobStore().list(container), new Predicate<StorageMetadata>() { @Override public boolean apply(StorageMetadata input) { return input.getType() == StorageType.BLOB; } }); assertEquals(Iterables.size(listing), 0, String.format( "deleting %s, we still have %s blobs left in container %s, using encoding %s", name, Iterables .size(listing), container, LOCAL_ENCODING)); }
@Test public void testListAllWhenOnePage() throws Exception { BlobStore blobStore = createMock(BlobStore.class); ListContainerOptions options = ListContainerOptions.NONE; StorageMetadata v1 = createMock(StorageMetadata.class); PageSet<StorageMetadata> pageSet = new PageSetImpl<StorageMetadata>(ImmutableList.of(v1), null); EasyMock.<PageSet<? extends StorageMetadata>> expect(blobStore.list(containerName, options)).andReturn(pageSet) .once(); EasyMock.replay(blobStore); Iterable<StorageMetadata> iterable = BlobStores.listAll(blobStore, containerName, options); assertEquals(ImmutableList.copyOf(iterable), ImmutableList.of(v1)); }
@Test(groups = { "integration", "live" }) void containerDoesntExist() { Set<? extends StorageMetadata> list = view.getBlobStore().list(); assert !list.contains(new MutableStorageMetadataImpl()); }
@Override public void run() { PageSet<? extends StorageMetadata> list = view.getBlobStore().list(); assert Iterables.any(list, new Predicate<StorageMetadata>() { public boolean apply(StorageMetadata md) { return containerName.equals(md.getName()) && location.equals(md.getLocation()); } }) : String.format("container %s/%s not found in list %s", location, containerName, list); assertTrue(view.getBlobStore().containerExists(containerName), containerName); }
@Test(groups = { "integration", "live" }) public void testListContainer() throws InterruptedException, ExecutionException, TimeoutException { String containerName = getContainerName(); try { add15UnderRoot(containerName); Set<? extends StorageMetadata> container = view.getBlobStore().list(containerName); assertEquals(container.size(), 15); } finally { returnContainer(containerName); } }
private Set<Service> getServices(Credentials creds) throws Exception { BlobStoreContext tmpContext = BlobStoreContextFactoryImpl.INSTANCE.newBlobStoreContext(location); try { tmpContext.getBlobStore().list(); LoadingCache<Credentials, Access> authCache = getAuthCache(tmpContext); Access tmpAccess = authCache.get(creds); return ImmutableSet.copyOf(tmpAccess); } finally { tmpContext.close(); } }
@Test(groups = { "integration", "live" }) public void testListContainerMaxResults() throws InterruptedException { String containerName = getContainerName(); try { addAlphabetUnderRoot(containerName); PageSet<? extends StorageMetadata> container = view.getBlobStore().list(containerName, maxResults(5)); assertEquals(container.size(), 5); assert container.getNextMarker() != null; } finally { returnContainer(containerName); } }