/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { init(); Collection<InetSocketAddress> addrs = new ArrayList<>(); try { Storage.Objects.List listObjects = storage.objects().list(bucketName); com.google.api.services.storage.model.Objects objects; do { objects = listObjects.execute(); if (objects == null || objects.getItems() == null) break; for (StorageObject object : objects.getItems()) addrs.add(addrFromString(object.getName())); listObjects.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); } catch (Exception e) { throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); } return addrs; }
Iterable<StorageObject> storageObjects = Iterables.concat( firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
private Iterator<String> createIterator() throws Exception { if (listObjectsSrvcHandle == null) initListing(); List<String> temp = Lists.newArrayList(); // a container of results // Sends the metadata request to the server and returns the parsed metadata response. this.objectsContainerHandle = listObjectsSrvcHandle.execute(); for (StorageObject object : this.objectsContainerHandle.getItems()) { // processing a page of results temp.add(object.getName()); } return temp.iterator(); }
@Override public boolean hasNext() { if (this.iterator.hasNext()) { return true; } while (this.objectsContainerHandle.getNextPageToken() != null && !iterator.hasNext()) try { // if here, you have iterated through all elements of the previous page, now, get // the next page of results this.listObjectsSrvcHandle.setPageToken(objectsContainerHandle.getNextPageToken()); this.iterator = createIterator(); } catch (Exception e) { throw new RuntimeException( "Exception encountered fetching elements, see previous messages for details.", e); } return this.iterator.hasNext(); }
@Test public void testMatch() throws Exception { Objects modelObjects = new Objects(); List<StorageObject> items = new ArrayList<>(); items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L /* fileSize */)); modelObjects.setItems(items); when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class))) .thenReturn(modelObjects);
o = new Objects(){ @Override public void myAbstractMethod(){ ... } }
private StorageObject resolveSingularItem(ObjectType objectType, String objectKey, GoogleNamedAccountCredentials credentials, Storage storage, String bucketName) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; try { Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects = objectsList.execute(); List<StorageObject> items = objects.getItems(); if (items != null && items.size() == 1) { return items.get(0); } else { throw new IllegalArgumentException("Unable to resolve singular " + objectType + " at " + daoRoot(credentials, objectType.getGroup()) + '/' + objectKey + "."); } } catch (IOException e) { throw new IllegalArgumentException("Could not fetch items from Google Cloud Storage: " + e.getMessage(), e); } }
do { objects = listObjects.execute(); List<StorageObject> items = objects.getItems(); if (items != null) { for (StorageObject item : items) { } while (objects.getNextPageToken() != null);
@Override public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setFields(FIELDS.getString(options)) .execute(); Iterable<StorageObject> storageObjects = Iterables.concat( firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { throw translate(ex); } }
/** * Lists the objects in a bucket * * @param bucketName bucket name to list * @return Array of object names * @throws Exception */ public static List<String> listBucket(String bucketName) throws Exception { Storage storage = getStorage(); List<String> list = new ArrayList<String>(); List<StorageObject> objects = storage.objects().list(bucketName).execute().getItems(); if(objects != null) { for(StorageObject o : objects) { list.add(o.getName()); } } return list; }
public static Iterable<StorageObject> list(Storage storage, String bucketName) throws IOException { List<List<StorageObject>> pagedList = Lists.newArrayList(); Storage.Objects.List listObjects = storage.objects().list(bucketName); Objects objects; do { objects = listObjects.execute(); List<StorageObject> items = objects.getItems(); if (items != null) { pagedList.add(objects.getItems()); } listObjects.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); return Iterables.concat(pagedList); }
List<String> pagePrefixes = items.getPrefixes(); if (pagePrefixes != null) { logger.atFine().log("listed %s prefixes", pagePrefixes.size()); List<StorageObject> objects = items.getItems(); if (objects != null) { logger.atFine().log("listed %s objects", objects.size()); listedPrefixes.addAll(prefixes); return items.getNextPageToken();
/** * Fetch a list of the objects within the given bucket. * * @param bucketName the name of the bucket to list. * @return a list of the contents of the specified bucket. */ public static List<StorageObject> listBucket(String bucketName) throws IOException, GeneralSecurityException { Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List<StorageObject> results = new ArrayList<StorageObject>(); Objects objects; // Iterate through each page of results, and add them to our results list. do { objects = listRequest.execute(); // Add the items in this page of results to the list we'll return. results.addAll(objects.getItems()); // Get the next page, in the next iteration of this loop. listRequest.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); return results; } // [END list_bucket]
Iterable<StorageObject> storageObjects = Iterables.concat( firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
List<StorageObject> items = objects.getItems(); objectsList.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); } catch (IOException e) { log.error("Could not fetch items from Google Cloud Storage: {}", e);
List<StorageObject> items = objects.getItems(); if (items != null) { for (StorageObject obj : items) { listMethod.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); executor.shutdown();
do { Objects objects = options.getGcsUtil().listObjects(gcsPattern.getBucket(), prefix, pageToken); if (objects.getItems() == null) { break; for (StorageObject o : objects.getItems()) { String name = o.getName(); pageToken = objects.getNextPageToken(); } while (pageToken != null); return MatchResult.create(Status.OK, results);