private void initListing() { try { this.listObjectsSrvcHandle = objectsResoruceHandle.list(bucketName); // == list objects within bucket // fetch elements within bucket that matches this prefix this.listObjectsSrvcHandle.setPrefix(this.prefix); } catch (IOException e) { throw new RuntimeException("Unable to get gcslist handle to bucket: " + bucketName, e); } }
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { init(); Collection<InetSocketAddress> addrs = new ArrayList<>(); try { Storage.Objects.List listObjects = storage.objects().list(bucketName); com.google.api.services.storage.model.Objects objects; do { objects = listObjects.execute(); if (objects == null || objects.getItems() == null) break; for (StorageObject object : objects.getItems()) addrs.add(addrFromString(object.getName())); listObjects.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); } catch (Exception e) { throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); } return addrs; }
storage .objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(Option.VERSIONS.getBoolean(options))
when(mockStorageObjects.get("testbucket", "testdirectory/otherfile")) .thenReturn(mockStorageGet); when(mockStorageObjects.list("testbucket")).thenReturn(mockStorageList); when(mockStorageGet.execute()) .thenReturn(new StorageObject().setBucket("testbucket").setName("testdirectory/otherfile"));
when(mockStorageObjects.get("testbucket", "test/directory/otherfile")) .thenReturn(mockStorageGet); when(mockStorageObjects.list("testbucket")).thenReturn(mockStorageList); when(mockStorageGet.execute()) .thenReturn(
storage .objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(Option.VERSIONS.getBoolean(options))
public void visitObjects(String bucketName, String pathPrefix, VisitorOperation op) throws IOException { Storage.Objects.List listMethod = storage_.objects().list(bucketName); listMethod.setPrefix(pathPrefix); Objects objects;
private Storage.Objects.List createListRequest( String bucketName, String objectNamePrefix, String delimiter, boolean includeTrailingDelimiter, long maxResults) throws IOException { logger.atFine().log( "createListRequest(%s, %s, %s, %s, %d)", bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults); checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty"); Storage.Objects.List listObject = configureRequest(gcs.objects().list(bucketName), bucketName); // Set delimiter if supplied. if (delimiter != null) { listObject.setDelimiter(delimiter); listObject.setIncludeTrailingDelimiter(includeTrailingDelimiter); } // Set number of items to retrieve per call. if (maxResults <= 0 || maxResults + 1 >= storageOptions.getMaxListItemsPerCall()) { listObject.setMaxResults(storageOptions.getMaxListItemsPerCall()); } else { // We add one in case we filter out objectNamePrefix. listObject.setMaxResults(maxResults + 1); } // Set prefix if supplied. if (!Strings.isNullOrEmpty(objectNamePrefix)) { listObject.setPrefix(objectNamePrefix); } return listObject; }
@Override public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) { try { Objects objects = storage.objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(VERSIONS.getBoolean(options)) .setDelimiter(DELIMITER.getString(options)) .setPrefix(PREFIX.getString(options)) .setMaxResults(MAX_RESULTS.getLong(options)) .setPageToken(PAGE_TOKEN.getString(options)) .setFields(FIELDS.getString(options)) .execute(); Iterable<StorageObject> storageObjects = Iterables.concat( firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { throw translate(ex); } }
/** Lists {@link Objects} given the {@code bucket}, {@code prefix}, {@code pageToken}. */ public Objects listObjects(String bucket, String prefix, @Nullable String pageToken) throws IOException { // List all objects that start with the prefix (including objects in sub-directories). Storage.Objects.List listObject = storageClient.objects().list(bucket); listObject.setMaxResults(MAX_LIST_ITEMS_PER_CALL); listObject.setPrefix(prefix); if (pageToken != null) { listObject.setPageToken(pageToken); } try { return ResilientOperation.retry( ResilientOperation.getGoogleRequestCallable(listObject), createBackOff(), RetryDeterminer.SOCKET_ERRORS, IOException.class); } catch (Exception e) { throw new IOException( String.format("Unable to match files in bucket %s, prefix %s.", bucket, prefix), e); } }
@Test public void testMatchingAgainstRequestType() throws Exception { GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class); options.setGcpCredential(new TestCredential()); options.setGoogleApiTrace( new GoogleApiTracer() .addTraceFor( Transport.newStorageClient(options) .build() .objects() .get("aProjectId", "aObjectId"), "TraceDestination")); Storage.Objects.Get getRequest = Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); assertEquals("TraceDestination", getRequest.get("$trace")); Storage.Objects.List listRequest = Transport.newStorageClient(options).build().objects().list("testProjectId"); assertNull(listRequest.get("$trace")); }
/** * Lists the objects in a bucket * * @param bucketName bucket name to list * @return Array of object names * @throws Exception */ public static List<String> listBucket(String bucketName) throws Exception { Storage storage = getStorage(); List<String> list = new ArrayList<String>(); List<StorageObject> objects = storage.objects().list(bucketName).execute().getItems(); if(objects != null) { for(StorageObject o : objects) { list.add(o.getName()); } } return list; }
@Test public void testMatchingAllCalls() throws Exception { String[] args = new String[] {STORAGE_TRACE}; GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); options.setGcpCredential(new TestCredential()); assertNotNull(options.getGoogleApiTrace()); Storage.Objects.Get getRequest = Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); assertEquals("TraceDestination", getRequest.get("$trace")); Storage.Objects.List listRequest = Transport.newStorageClient(options).build().objects().list("testProjectId"); assertEquals("TraceDestination", listRequest.get("$trace")); }
@Test public void testWithMultipleTraces() throws Exception { String[] args = new String[] {STORAGE_GET_AND_LIST_TRACE}; GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); options.setGcpCredential(new TestCredential()); assertNotNull(options.getGoogleApiTrace()); Storage.Objects.Get getRequest = Transport.newStorageClient(options).build().objects().get("testBucketId", "testObjectId"); assertEquals("GetTraceDestination", getRequest.get("$trace")); Storage.Objects.List listRequest = Transport.newStorageClient(options).build().objects().list("testProjectId"); assertEquals("ListTraceDestination", listRequest.get("$trace")); }
@Test public void testWhenTracingDoesNotMatch() throws Exception { String[] args = new String[] {STORAGE_GET_TRACE}; GcsOptions options = PipelineOptionsFactory.fromArgs(args).as(GcsOptions.class); options.setGcpCredential(new TestCredential()); assertNotNull(options.getGoogleApiTrace()); Storage.Objects.List request = Transport.newStorageClient(options).build().objects().list("testProjectId"); assertNull(request.get("$trace")); }
Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects;
/** * Fetch a list of the objects within the given bucket. * * @param bucketName the name of the bucket to list. * @return a list of the contents of the specified bucket. */ public static List<StorageObject> listBucket(String bucketName) throws IOException, GeneralSecurityException { Storage client = StorageFactory.getService(); Storage.Objects.List listRequest = client.objects().list(bucketName); List<StorageObject> results = new ArrayList<StorageObject>(); Objects objects; // Iterate through each page of results, and add them to our results list. do { objects = listRequest.execute(); // Add the items in this page of results to the list we'll return. results.addAll(objects.getItems()); // Get the next page, in the next iteration of this loop. listRequest.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); return results; } // [END list_bucket]
public static Iterable<StorageObject> list(Storage storage, String bucketName) throws IOException { List<List<StorageObject>> pagedList = Lists.newArrayList(); Storage.Objects.List listObjects = storage.objects().list(bucketName); Objects objects; do { objects = listObjects.execute(); List<StorageObject> items = objects.getItems(); if (items != null) { pagedList.add(objects.getItems()); } listObjects.setPageToken(objects.getNextPageToken()); } while (objects.getNextPageToken() != null); return Iterables.concat(pagedList); }
private StorageObject resolveSingularItem(ObjectType objectType, String objectKey, GoogleNamedAccountCredentials credentials, Storage storage, String bucketName) { String rootFolder = daoRoot(credentials, objectType.getGroup()) + "/" + objectKey; try { Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects = objectsList.execute(); List<StorageObject> items = objects.getItems(); if (items != null && items.size() == 1) { return items.get(0); } else { throw new IllegalArgumentException("Unable to resolve singular " + objectType + " at " + daoRoot(credentials, objectType.getGroup()) + '/' + objectKey + "."); } } catch (IOException e) { throw new IllegalArgumentException("Could not fetch items from Google Cloud Storage: " + e.getMessage(), e); } }
public Storage.Objects.List list(final String bucket) throws IOException { return storage.objects().list(bucket); } }