blobStore.list(bucketName, new ListContainerOptions().recursive());
public synchronized void downloadNewFiles(File downloadDir) { log.info("Downloading new files since {}", lastMarker); BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/ BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); //Gets key within the time range specified NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages); //Download only for keys within that range for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) { log.info("Downloading file: " + blobMeta.getValue()); downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue())); lastMarker = blobMeta.getValue(); synchronized (CloudFilesManager.this) { // this is where we resume from. MarkerUtils.writeLastMarker(blobMeta.getValue()); } } log.info("Updated the last marker value as " + lastMarker); }
public synchronized boolean hasNewFiles() { // see if there are any files since lastMarker. BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker); boolean emptiness = getBlobsWithinRange(pages).isEmpty(); if(emptiness) { log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME)); } else { log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME)); } return !emptiness; }
String containerName) throws IOException, S3Exception { String blobStoreType = getBlobStoreType(blobStore); ListContainerOptions options = new ListContainerOptions(); String encodingType = request.getParameter("encoding-type"); String delimiter = request.getParameter("delimiter");
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#recursive() */ public static ListContainerOptions recursive() { ListContainerOptions options = new ListContainerOptions(); return options.recursive(); }
/** * @see ListContainerOptions#withDetails() */ public static ListContainerOptions withDetails() { ListContainerOptions options = new ListContainerOptions(); return options.withDetails(); } }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#inDirectory(String) */ public static ListContainerOptions inDirectory(String directory) { ListContainerOptions options = new ListContainerOptions(); return options.inDirectory(directory); }
/** * @see ListContainerOptions#delimiter(String) */ public static ListContainerOptions delimiter(String delimiterString) { ListContainerOptions options = new ListContainerOptions(); return options.delimiter(delimiterString); } }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
@Test public void testNullMarker() { ListContainerOptions options = new ListContainerOptions(); assertEquals(options.getMarker(), null); }
@Test public void testNullMaxResults() { ListContainerOptions options = new ListContainerOptions(); assertEquals(options.getMaxResults(), null); }
@Test public void testNullPath() { ListContainerOptions options = new ListContainerOptions(); assertEquals(options.getDir(), null); }
@Test public void testMaxResults() { ListContainerOptions options = new ListContainerOptions(); options.maxResults(1000); assertEquals(options.getMaxResults(), Integer.valueOf(1000)); }
@Test public void testMarker() { ListContainerOptions options = new ListContainerOptions(); options.afterMarker("test"); assertEquals(options.getMarker(), "test"); }
@Test public void testPath() { ListContainerOptions options = new ListContainerOptions(); options.inDirectory("test"); assertEquals(options.getDir(), "test"); }
@Test public void testRecursive() { ListContainerOptions options = new ListContainerOptions(); options.recursive(); assertTrue(options.isRecursive()); }
@Test public void testTwoOptions() { ListContainerOptions options = new ListContainerOptions(); options.inDirectory("test").maxResults(1); assertEquals(options.getDir(), "test"); assertEquals(options.getMaxResults(), Integer.valueOf(1)); }
public void testLargerThanOnePageNoOptions() { blobstore.createContainerInLocation(null, "goodies"); for (int i = 0; i < 1001; i++) { blobstore.putBlob("goodies", blobstore.blobBuilder(i + "").payload(i + "").build()); } Iterable<? extends StorageMetadata> listing = concatter.execute("goodies", new ListContainerOptions()); assertEquals(Iterables.size(listing), 1001); }