Refine search
PageSet<? extends StorageMetadata> set = blobStore.list(containerName, options); if (Quirks.OPAQUE_MARKERS.contains(blobStoreType)) { lastKeyToMarker.put(Maps.immutableEntry(containerName, Iterables.getLast(set).getName()), nextMarker); switch (metadata.getType()) { case FOLDER: commonPrefixes.add(metadata.getName()); continue; default: metadata.getName())); Date lastModified = metadata.getLastModified(); if (lastModified != null) { writeSimpleElement(xml, "LastModified", String eTag = metadata.getETag(); if (eTag != null) { writeSimpleElement(xml, "ETag", maybeQuoteETag(eTag)); String.valueOf(metadata.getSize())); writeSimpleElement(xml, "StorageClass", StorageClass.fromTier(metadata.getTier()).toString());
public StorageMetadata apply(StorageMetadata md) { checkNotNull(md, "metadata"); String name = checkNotNull(md.getName(), "metadata.name"); for (String suffix : BlobStoreConstants.DIRECTORY_SUFFIXES) { if (name.endsWith(suffix)) name = name.substring(0, name.length() - suffix.length()); } return new StorageMetadataImpl(StorageType.RELATIVE_PATH, md.getProviderId(), name, md.getLocation(), md.getUri(), md.getETag(), md.getCreationDate(), md.getLastModified(), md.getUserMetadata()); }
private void handleContainerList(HttpServletResponse response, BlobStore blobStore) throws IOException { PageSet<? extends StorageMetadata> buckets = blobStore.list(); xml.writeStartElement("Bucket"); writeSimpleElement(xml, "Name", metadata.getName()); Date creationDate = metadata.getCreationDate(); if (creationDate == null) { blobStore.getContext().utils().date() .iso8601DateFormat(creationDate).trim());
public MutableStorageMetadataImpl(StorageMetadata from) { super(from); this.eTag = from.getETag(); this.lastModified = from.getLastModified(); }
public PageSet<? extends StorageMetadata> apply(ListBucketResponse from) { Set<StorageMetadata> contents = Sets.<StorageMetadata> newHashSet(Iterables.transform(from, object2blobMd)); Map<String, StorageMetadata> nameToMd = Maps.uniqueIndex(contents, indexer); for (String prefix : from.getCommonPrefixes()) { prefix = prefix.endsWith("/") ? prefix.substring(0, prefix.lastIndexOf('/')) : prefix; if (!nameToMd.containsKey(prefix) || nameToMd.get(prefix).getType() != StorageType.RELATIVE_PATH) contents.add(prefix2ResourceMd.apply(prefix)); } return new PageSetImpl<StorageMetadata>(contents, from.getNextMarker()); } }
@Override public Set<? extends BlobMetadata> execute(final String containerName, final ListContainerOptions options) { final List<Iterable<? extends BlobMetadata>> lists = newArrayList(); Iterable<? extends StorageMetadata> pwdList = lister.execute(containerName, options); for (StorageMetadata md : filter(pwdList, new Predicate<StorageMetadata>() { @Override public boolean apply(StorageMetadata input) { return (input.getType() == StorageType.FOLDER || input.getType() == StorageType.RELATIVE_PATH) && options.isRecursive(); } })) { String directory = (options.getDir() != null) ? options.getDir() + "/" + md.getName() : md.getName(); lists.add(execute(containerName, options.clone().inDirectory(directory))); } lists.add(transform(filter(pwdList, new Predicate<StorageMetadata>() { @Override public boolean apply(StorageMetadata input) { return input.getType() == StorageType.BLOB; } }), new Function<StorageMetadata, BlobMetadata>() { @Override public BlobMetadata apply(StorageMetadata from) { return (BlobMetadata) from; } })); return newLinkedHashSet(concat(lists)); } }
SortedSet<StorageMetadata> contents = newTreeSet(transform(blobBelongingToContainer, new Function<String, StorageMetadata>() { public StorageMetadata apply(String key) { if (options.getMarker() != null) { final String finalMarker = options.getMarker(); StorageMetadata lastMarkerMetadata = find(contents, new Predicate<StorageMetadata>() { public boolean apply(StorageMetadata metadata) { return metadata.getName().compareTo(finalMarker) > 0; final String prefix = options.getDir(); if (prefix != null) { contents = newTreeSet(filter(contents, new Predicate<StorageMetadata>() { if (!contents.isEmpty()) { StorageMetadata lastElement = contents.last(); contents = newTreeSet(Iterables.limit(contents, maxResults)); if (!contents.contains(lastElement)) { marker = contents.last().getName(); if (!options.isDetailed()) { for (StorageMetadata md : contents) { md.getUserMetadata().clear();
/** * Tries to delete all containers, runs up to two times */ protected static void deleteEverything(final BlobStoreContext context) throws Exception { try { for (int i = 0; i < 2; i++) { Iterable<? extends StorageMetadata> testContainers = Iterables.filter(context.getBlobStore().list(), new Predicate<StorageMetadata>() { public boolean apply(StorageMetadata input) { return (input.getType() == StorageType.CONTAINER || input.getType() == StorageType.FOLDER) && input.getName().startsWith(CONTAINER_PREFIX); } }); for (StorageMetadata container : testContainers) { deleteContainerOrWarnIfUnable(context, container.getName()); } } // try twice } catch (CancellationException e) { throw e; } }
public void testListDirectory() { String containerName = "testListDir"; String directory = "dir"; blobStore.createContainerInLocation(null, containerName); blobStore.createDirectory(containerName, directory); blobStore.putBlob(containerName, blobStore.blobBuilder(directory + "/foo").payload("").build()); blobStore.putBlob(containerName, blobStore.blobBuilder(directory + "/bar").payload("").build()); Iterable<? extends StorageMetadata> results = concatter.execute(containerName, ListContainerOptions.NONE); assertThat(results).hasSize(2); assertThat(Iterables.get(results, 0).getName()).isEqualTo(directory); assertThat(Iterables.get(results, 0).getType()).isEqualTo(StorageType.FOLDER); assertThat(Iterables.get(results, 1).getName()).isEqualTo(directory + '/'); assertThat(Iterables.get(results, 1).getType()).isEqualTo(StorageType.RELATIVE_PATH); }
public void testDelimiterList() { String container = "delimiter"; String delimiter = "--"; blobStore.createContainerInLocation(null, container); blobStore.putBlob(container, blobStore.blobBuilder("foo").payload("").build()); blobStore.putBlob(container, blobStore.blobBuilder("other" + delimiter + "bar").payload("").build()); Iterable<? extends StorageMetadata> results = blobStore.list(container, ListContainerOptions.Builder.delimiter(delimiter)); assertEquals(Iterables.size(results), 2); assertEquals(Iterables.get(results, 0).getType(), StorageType.BLOB); assertEquals(Iterables.get(results, 0).getName(), "foo"); assertEquals(Iterables.get(results, 1).getType(), StorageType.RELATIVE_PATH); assertEquals(Iterables.get(results, 1).getName(), "other" + delimiter); } }
public void testListWithPrefixAndDelimiter() { String containerName = "prefixWithSeparator"; String prefix = "foo"; blobStore.createContainerInLocation(null, containerName); blobStore.putBlob(containerName, blobStore.blobBuilder(prefix + "-object").payload("").build()); blobStore.putBlob(containerName, blobStore.blobBuilder(prefix + "bar-object").payload("") .build()); blobStore.putBlob(containerName, blobStore.blobBuilder(prefix + "baz-object").payload("") .build()); blobStore.putBlob(containerName, blobStore.blobBuilder("bar-object").payload("").build()); Iterable<? extends StorageMetadata> results = concatter.execute(containerName, ListContainerOptions.Builder.prefix(prefix).delimiter("-")); assertThat(Iterables.size(results)).isEqualTo(3); assertThat(Iterables.get(results, 0).getType()).isEqualTo(StorageType.RELATIVE_PATH); assertThat(Iterables.get(results, 0).getName()).isEqualTo(prefix + "-"); assertThat(Iterables.get(results, 1).getName()).isEqualTo(prefix + "bar-"); assertThat(Iterables.get(results, 1).getType()).isEqualTo(StorageType.RELATIVE_PATH); assertThat(Iterables.get(results, 2).getName()).isEqualTo(prefix + "baz-"); assertThat(Iterables.get(results, 2).getType()).isEqualTo(StorageType.RELATIVE_PATH); }
public void testDirectoryListing() { String containerName = "testDirectoryListing"; blobStore.createContainerInLocation(null, containerName); blobStore.createDirectory(containerName, "dir"); blobStore.createDirectory(containerName, "dir/dir"); PageSet<? extends StorageMetadata> results = blobStore.list(containerName); assertThat(results.size()).isEqualTo(2); assertThat(Iterables.get(results, 0).getName()).isEqualTo("dir"); assertThat(Iterables.get(results, 1).getName()).isEqualTo("dir/"); results = blobStore.list(containerName, ListContainerOptions.Builder.inDirectory("dir")); assertThat(results.size()).isEqualTo(1); assertThat(Iterables.get(results, 0).getName()).isEqualTo("dir/dir"); assertThat(Iterables.get(results, 0).getType()).isEqualTo(StorageType.FOLDER); blobStore.putBlob(containerName, blobStore.blobBuilder("dir/dir/blob").payload("").build()); results = blobStore.list(containerName, ListContainerOptions.Builder.inDirectory("dir")); assertThat(results.size()).isEqualTo(2); assertThat(Iterables.get(results, 0).getName()).isEqualTo("dir/dir"); assertThat(Iterables.get(results, 0).getType()).isEqualTo(StorageType.FOLDER); assertThat(Iterables.get(results, 1).getName()).isEqualTo("dir/dir/"); assertThat(Iterables.get(results, 1).getType()).isEqualTo(StorageType.RELATIVE_PATH); } }
@Override public List<MultipartUpload> listMultipartUploads(String container) { ImmutableList.Builder<MultipartUpload> mpus = ImmutableList.builder(); ListContainerOptions options = new ListContainerOptions().prefix(MULTIPART_PREFIX).recursive(); int uuidLength = UUID.randomUUID().toString().length(); while (true) { PageSet<? extends StorageMetadata> pageSet = list(container, options); for (StorageMetadata sm : pageSet) { if (!sm.getName().endsWith("-stub")) { continue; } String uploadId = sm.getName().substring(MULTIPART_PREFIX.length(), MULTIPART_PREFIX.length() + uuidLength); String blobName = sm.getName().substring(MULTIPART_PREFIX.length() + uuidLength + 1); int index = blobName.lastIndexOf('-'); blobName = blobName.substring(0, index); mpus.add(MultipartUpload.create(container, blobName, uploadId, null, null)); } if (pageSet.isEmpty() || pageSet.getNextMarker() == null) { break; } options.afterMarker(pageSet.getNextMarker()); } return mpus.build(); }
try { BlobStore blobStore = blobStoreContext.getBlobStore(); ListContainerOptions listContainerOptions = new ListContainerOptions(); if (listRecursiveFlag == ListRecursiveFlag.TRUE) { listContainerOptions.recursive(); listContainerOptions.prefix(prefix); if (listRecursiveFlag == ListRecursiveFlag.FALSE) { listContainerOptions.delimiter(BucketName.BUCKET_SEPARATOR); LOGGER.debug("list container:" + objectHandle.getContainer() + " prefix:" + listContainerOptions.getPrefix() + " del:" + listContainerOptions.getDelimiter()); org.jclouds.blobstore.domain.PageSet<? extends org.jclouds.blobstore.domain.StorageMetadata> ps = blobStore.list(objectHandle.getContainer(), listContainerOptions); LinkedHashSet<StorageMetadata> set = new LinkedHashSet<StorageMetadata>(); for (org.jclouds.blobstore.domain.StorageMetadata s : ps) { StorageType type = s.getType()==null?null:StorageType.valueOf(s.getType().name()); Location location = copyLocation(new HashSet<String>(), s.getLocation()); StorageMetadata e = new StorageMetadataImpl(type, s.getProviderId(), s.getName(), location, s.getUri(), s.getETag(), s.getCreationDate(), s.getLastModified(), s.getUserMetadata(), s.getSize()); set.add(e); return new PageSetImpl<>(set, ps.getNextMarker());
/** Test that listing with a marker prefix matches the first key with that prefix. */ @Test public void testListMarkerPrefix() throws Exception { BlobStore blobStore = view.getBlobStore(); final String container = getContainerName(); try { blobStore.createContainerInLocation(null, container); blobStore.putBlob(container, blobStore.blobBuilder("a/a").payload("").build()); blobStore.putBlob(container, blobStore.blobBuilder("b/b").payload("").build()); ListContainerOptions options = new ListContainerOptions().afterMarker("b/").recursive(); PageSet<? extends StorageMetadata> res = blobStore.list(container, options); assertThat(res).hasSize(1); assertThat(res.iterator().next().getName()).isEqualTo("b/b"); } finally { returnContainer(container); } }
public DirectoryEntry apply(StorageMetadata from) { FileType type = (from.getType() == StorageType.FOLDER || from.getType() == StorageType.RELATIVE_PATH) ? FileType.DIRECTORY : FileType.REGULAR; return new DirectoryEntry(from.getProviderId(), type, from.getName()); }
public MutableObjectInfoWithMetadata apply(StorageMetadata from) { if (from == null) return null; MutableObjectInfoWithMetadata to = new MutableObjectInfoWithMetadataImpl(); if (from.getType() == StorageType.BLOB) { to.setContentType(((BlobMetadata) from).getContentMetadata().getContentType()); to.setBytes(((BlobMetadata) from).getContentMetadata().getContentLength()); to.setHash(((BlobMetadata) from).getContentMetadata().getContentMD5()); } else if (from.getType() == StorageType.RELATIVE_PATH) { to.setContentType("application/directory"); } if (from.getETag() != null && to.getHash() == null) to.setHash(base16().lowerCase().decode(from.getETag())); to.setName(from.getName()); to.setLastModified(from.getLastModified()); if (from.getUserMetadata() != null) { for (Entry<String, String> entry : from.getUserMetadata().entrySet()) to.getMetadata().put(entry.getKey().toLowerCase(), entry.getValue()); } return to; }
@Test(dataProvider = "ignoreOnMacOSX") public void testListDirectoryBlobsS3FS() { blobStore.createContainerInLocation(null, CONTAINER_NAME); checkForContainerContent(CONTAINER_NAME, null); String d = TestUtils.createRandomBlobKey("directory-", ""); blobStore.putBlob(CONTAINER_NAME, createDirBlob(d + File.separator)); assertTrue(blobStore.blobExists(CONTAINER_NAME, d + File.separator)); ListContainerOptions options = ListContainerOptions.Builder .withDetails() .inDirectory(""); PageSet<? extends StorageMetadata> res = blobStore.list(CONTAINER_NAME, options); assertTrue(res.size() == 1); assertEquals(res.iterator().next().getName(), d + File.separator); }
@Override public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) { ImmutableList.Builder<MultipartPart> parts = ImmutableList.builder(); PageSet<? extends StorageMetadata> pageSet = list(mpu.containerName(), new ListContainerOptions().prefix(mpu.id() + "/")); // TODO: pagination for (StorageMetadata sm : pageSet) { int lastSlash = sm.getName().lastIndexOf('/'); int partNumber = Integer.parseInt(sm.getName().substring(lastSlash + 1)); parts.add(MultipartPart.create(partNumber, sm.getSize(), sm.getETag(), sm.getLastModified())); } return parts.build(); }
private Set<String> listAllBlobs() { Set<String> blobNames = new HashSet<String>(); ListContainerOptions options = new ListContainerOptions(); while (true) { PageSet<? extends StorageMetadata> set = blobStoreRead.list( containerName, options); for (StorageMetadata sm : set) { blobNames.add(sm.getName()); } String marker = set.getNextMarker(); if (marker == null) { break; } options = options.afterMarker(marker); } return blobNames; }