public synchronized void downloadNewFiles(File downloadDir) { log.info("Downloading new files since {}", lastMarker); BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/ BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); //Gets key within the time range specified NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages); //Download only for keys within that range for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) { log.info("Downloading file: " + blobMeta.getValue()); downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue())); lastMarker = blobMeta.getValue(); synchronized (CloudFilesManager.this) { // this is where we resume from. MarkerUtils.writeLastMarker(blobMeta.getValue()); } } log.info("Updated the last marker value as " + lastMarker); }
public synchronized boolean hasNewFiles() { // see if there are any files since lastMarker. BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker); boolean emptiness = getBlobsWithinRange(pages).isEmpty(); if(emptiness) { log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME)); } else { log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME)); } return !emptiness; }
options.afterMarker(marker);
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
/** * @see ListContainerOptions#afterMarker(String) */ public static ListContainerOptions afterMarker(String marker) { ListContainerOptions options = new ListContainerOptions(); return options.afterMarker(marker); }
private void advance() { if (marker != null) { options.afterMarker(marker); } PageSet<? extends StorageMetadata> set = blobStore.list( containerName, options); marker = set.getNextMarker(); iterator = set.iterator(); }
private Set<String> listAllBlobs() { Set<String> blobNames = new HashSet<String>(); ListContainerOptions options = new ListContainerOptions(); while (true) { PageSet<? extends StorageMetadata> set = blobStoreRead.list( containerName, options); for (StorageMetadata sm : set) { blobNames.add(sm.getName()); } String marker = set.getNextMarker(); if (marker == null) { break; } options = options.afterMarker(marker); } return blobNames; }
@Test public void testMarker() { ListContainerOptions options = new ListContainerOptions(); options.afterMarker("test"); assertEquals(options.getMarker(), "test"); }
@Test public void testMarker() { ListContainerOptions options = new ListContainerOptions(); options.afterMarker("test"); assertEquals(options.getMarker(), "test"); }
@Override public boolean hasNext() { if ((set == null) || (queue == null)) { set = store.list(cloudContainer, maxResults(BATCH)); loadElements(set); } if (!queue.isEmpty()) { return true; } else if (set.getNextMarker() != null) { set = store.list(cloudContainer, maxResults(BATCH).afterMarker(set.getNextMarker())); loadElements(set); if (!queue.isEmpty()) { return true; } } return false; }
@Override public boolean hasNext() { if ((set == null) || (queue == null)) { set = store.list(cloudContainer, maxResults(BATCH)); loadElements(set); } if (!queue.isEmpty()) { return true; } else if (set.getNextMarker() != null) { set = store.list(cloudContainer, maxResults(BATCH).afterMarker(set.getNextMarker())); loadElements(set); if (!queue.isEmpty()) { return true; } } return false; }
/** Test that listing with a marker prefix matches the first key with that prefix. */ @Test public void testListMarkerPrefix() throws Exception { BlobStore blobStore = view.getBlobStore(); final String container = getContainerName(); try { blobStore.createContainerInLocation(null, container); blobStore.putBlob(container, blobStore.blobBuilder("a/a").payload("").build()); blobStore.putBlob(container, blobStore.blobBuilder("b/b").payload("").build()); ListContainerOptions options = new ListContainerOptions().afterMarker("b/").recursive(); PageSet<? extends StorageMetadata> res = blobStore.list(container, options); assertThat(res).hasSize(1); assertThat(res.iterator().next().getName()).isEqualTo("b/b"); } finally { returnContainer(container); } }
public ListContainerOptions apply(org.jclouds.atmos.options.ListOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getToken() != null) { options.afterMarker(optionsList[0].getToken()); } if (optionsList[0].getLimit() != null) { options.maxResults(optionsList[0].getLimit()); } if (optionsList[0].metaIncluded()) { options.withDetails(); } } return options; } }
public ListContainerOptions apply( org.jclouds.openstack.swift.options.ListContainerOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getPath() != null && !optionsList[0].getPath().equals("")) { options.inDirectory(optionsList[0].getPath()); } if (optionsList[0].getPrefix() != null && !optionsList[0].getPrefix().equals("")) { options.inDirectory(optionsList[0].getPrefix()); options.recursive(); } if (optionsList[0].getMarker() != null) { options.afterMarker(optionsList[0].getMarker()); } options.maxResults(optionsList[0].getMaxResults()); } return options; } }
public ListContainerOptions apply(org.jclouds.atmos.options.ListOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getToken() != null) { options.afterMarker(optionsList[0].getToken()); } if (optionsList[0].getLimit() != null) { options.maxResults(optionsList[0].getLimit()); } if (optionsList[0].metaIncluded()) { options.withDetails(); } } return options; } }