.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(Option.VERSIONS.getBoolean(options)) .setDelimiter(Option.DELIMITER.getString(options)) .setPrefix(Option.PREFIX.getString(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) .setFields(Option.FIELDS.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); Iterable<StorageObject> storageObjects = Iterables.concat(
items = listObject.execute(); } catch (IOException e) { String resource = StorageResourceId.createReadableString(listObject.getBucket(), listObject.getPrefix()); if (errorExtractor.itemNotFound(e)) { logger.atFine().withCause(e).log( String objectNamePrefix = listObject.getPrefix(); boolean objectPrefixEndsWithDelimiter = !Strings.isNullOrEmpty(objectNamePrefix) && objectNamePrefix.endsWith(PATH_DELIMITER);
when(mockStorageGet.execute()) .thenReturn(new StorageObject().setBucket("testbucket").setName("testdirectory/otherfile")); when(mockStorageList.execute()).thenReturn(modelObjects);
if (pageToken != null) { logger.atFine().log("listStorageObjectsAndPrefixes: next page %s", pageToken); listObject.setPageToken(pageToken);
.thenReturn( new StorageObject().setBucket("testbucket").setName("test/directory/otherfile")); when(mockStorageList.execute()).thenReturn(modelObjects);
if (pageToken != null) { logger.atFine().log("listObjectInfoPage: next page %s", pageToken); listObject.setPageToken(pageToken);
public void visitObjects(String bucketName, String pathPrefix, VisitorOperation op) throws IOException { Storage.Objects.List listMethod = storage_.objects().list(bucketName); listMethod.setPrefix(pathPrefix); Objects objects; ExecutorService executor = Executors.newFixedThreadPool(8); objects = listMethod.execute(); List<StorageObject> items = objects.getItems(); if (items != null) { listMethod.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); executor.shutdown();
.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(Option.VERSIONS.getBoolean(options)) .setDelimiter(Option.DELIMITER.getString(options)) .setPrefix(Option.PREFIX.getString(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) .setFields(Option.FIELDS.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); Iterable<StorageObject> storageObjects = Iterables.concat(
private Storage.Objects.List createListRequest( String bucketName, String objectNamePrefix, String delimiter, boolean includeTrailingDelimiter, long maxResults) throws IOException { logger.atFine().log( "createListRequest(%s, %s, %s, %s, %d)", bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults); checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty"); Storage.Objects.List listObject = configureRequest(gcs.objects().list(bucketName), bucketName); // Set delimiter if supplied. if (delimiter != null) { listObject.setDelimiter(delimiter); listObject.setIncludeTrailingDelimiter(includeTrailingDelimiter); } // Set number of items to retrieve per call. if (maxResults <= 0 || maxResults + 1 >= storageOptions.getMaxListItemsPerCall()) { listObject.setMaxResults(storageOptions.getMaxListItemsPerCall()); } else { // We add one in case we filter out objectNamePrefix. listObject.setMaxResults(maxResults + 1); } // Set prefix if supplied. if (!Strings.isNullOrEmpty(objectNamePrefix)) { listObject.setPrefix(objectNamePrefix); } return listObject; }
listObjects.setMaxResults(maxResults); do { Objects objects = listObjects.execute(); List<StorageObject> items = objects.getItems(); listObjects.setPageToken(token); } while (token != null);
/** Lists {@link Objects} given the {@code bucket}, {@code prefix}, {@code pageToken}. */ public Objects listObjects(String bucket, String prefix, @Nullable String pageToken) throws IOException { // List all objects that start with the prefix (including objects in sub-directories). Storage.Objects.List listObject = storageClient.objects().list(bucket); listObject.setMaxResults(MAX_LIST_ITEMS_PER_CALL); listObject.setPrefix(prefix); if (pageToken != null) { listObject.setPageToken(pageToken); } try { return ResilientOperation.retry( ResilientOperation.getGoogleRequestCallable(listObject), createBackOff(), RetryDeterminer.SOCKET_ERRORS, IOException.class); } catch (Exception e) { throw new IOException( String.format("Unable to match files in bucket %s, prefix %s.", bucket, prefix), e); } }
@Override public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setFields(FIELDS.getString(options)) .execute(); Iterable<StorageObject> storageObjects = Iterables.concat( firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { throw translate(ex); } }
/** * Lists the objects in a bucket * * @param bucketName bucket name to list * @return Array of object names * @throws Exception */ public static List<String> listBucket(String bucketName) throws Exception { Storage storage = getStorage(); List<String> list = new ArrayList<String>(); List<StorageObject> objects = storage.objects().list(bucketName).execute().getItems(); if(objects != null) { for(StorageObject o : objects) { list.add(o.getName()); } } return list; }
Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects; objects = objectsList.execute(); List<StorageObject> items = objects.getItems(); objectsList.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); } catch (IOException e) {
/** * Fetch a list of the objects within the given bucket. * * @param bucketName the name of the bucket to list. * @return a list of the contents of the specified bucket. */ public static List<StorageObject> listBucket(String bucketName) throws IOException, GeneralSecurityException { Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List<StorageObject> results = new ArrayList<StorageObject>(); Objects objects; // Iterate through each page of results, and add them to our results list. do { objects = listRequest.execute(); // Add the items in this page of results to the list we'll return. results.addAll(objects.getItems()); // Get the next page, in the next iteration of this loop. listRequest.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); return results; } // [END list_bucket]
public static Iterable<StorageObject> list(Storage storage, String bucketName) throws IOException { List<List<StorageObject>> pagedList = Lists.newArrayList(); Storage.Objects.List listObjects = storage.objects().list(bucketName); Objects objects; do { objects = listObjects.execute(); List<StorageObject> items = objects.getItems(); if (items != null) { pagedList.add(objects.getItems()); } listObjects.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); return Iterables.concat(pagedList); }
private StorageObject resolveSingularItem(ObjectType objectType, String objectKey, GoogleNamedAccountCredentials credentials, Storage storage, String bucketName) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; try { Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects = objectsList.execute(); List<StorageObject> items = objects.getItems(); if (items != null && items.size() == 1) { return items.get(0); } else { throw new IllegalArgumentException("Unable to resolve singular " + objectType + " at " + daoRoot(credentials, objectType.getGroup()) + '/' + objectKey + "."); } } catch (IOException e) { throw new IllegalArgumentException("Could not fetch items from Google Cloud Storage: " + e.getMessage(), e); } }
@Override public boolean hasNext() { if (this.iterator.hasNext()) { return true; } while (this.objectsContainerHandle.getNextPageToken() != null && !iterator.hasNext()) try { // if here, you have iterated through all elements of the previous page, now, get // the next page of results this.listObjectsSrvcHandle.setPageToken(objectsContainerHandle.getNextPageToken()); this.iterator = createIterator(); } catch (Exception e) { throw new RuntimeException( "Exception encountered fetching elements, see previous messages for details.", e); } return this.iterator.hasNext(); }
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { init(); Collection<InetSocketAddress> addrs = new ArrayList<>(); try { Storage.Objects.List listObjects = storage.objects().list(bucketName); com.google.api.services.storage.model.Objects objects; do { objects = listObjects.execute(); if (objects == null || objects.getItems() == null) break; for (StorageObject object : objects.getItems()) addrs.add(addrFromString(object.getName())); listObjects.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); } catch (Exception e) { throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); } return addrs; }