Refine search
/** * Deletes a bucket, even if non-empty. Objects in the bucket are listed and deleted until bucket * deletion succeeds or {@code timeout} expires. To allow for the timeout, this method uses a * separate thread to send the delete requests. Use {@link #forceDelete(Storage storage, String * bucket)} if spawning an additional thread is undesirable, such as in the App Engine production * runtime. * * @param storage the storage service to be used to issue requests * @param bucket the bucket to be deleted * @param timeout the maximum time to wait * @param unit the time unit of the timeout argument * @return true if deletion succeeded, false if timeout expired * @throws InterruptedException if the thread deleting the bucket is interrupted while waiting * @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects */ public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException { return forceDelete(storage, bucket, timeout, unit, ""); }
@BeforeClass public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { remoteStorageHelper = RemoteStorageHelper.create(); topicAdminClient = TopicAdminClient.create(); storageService = remoteStorageHelper.getOptions().getService(); notificationService = new DefaultNotificationFactory().create(remoteStorageHelper.getOptions().getService()); storageService.create(BucketInfo.of(BUCKET)); projectId = ServiceOptions.getDefaultProjectId(); }
/** * Creates a {@code RemoteStorageHelper} object using default project id and authentication * credentials. */ public static RemoteStorageHelper create() throws StorageHelperException { HttpTransportOptions transportOptions = StorageOptions.getDefaultHttpTransportOptions(); transportOptions = transportOptions.toBuilder().setConnectTimeout(60000).setReadTimeout(60000).build(); StorageOptions storageOptions = StorageOptions.newBuilder() .setRetrySettings(retrySettings()) .setTransportOptions(transportOptions) .build(); return new RemoteStorageHelper(storageOptions); }
@AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { // In beforeClass, we make buckets auto-delete blobs older than a day old. // Here, delete all buckets older than 2 days. They should already be empty and easy. long cleanTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); long cleanTimeout = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1); RemoteStorageHelper.cleanBuckets(storage, cleanTime, cleanTimeout); boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 1, TimeUnit.MINUTES); if (!wasDeleted && log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); } } }
@BeforeClass public static void beforeClass() throws IOException { // loads the credentials from local disk as par README RemoteStorageHelper gcsHelper = RemoteStorageHelper.create(); storageOptions = gcsHelper.getOptions(); project = storageOptions.getProjectId(); storage = storageOptions.getService(); // create and populate test bucket storage.create(BucketInfo.of(BUCKET)); fillFile(storage, BUCKET, SML_FILE, SML_SIZE); fillFile(storage, BUCKET, BIG_FILE, BIG_SIZE); BucketInfo requesterPaysBucket = BucketInfo.newBuilder(REQUESTER_PAYS_BUCKET).setRequesterPays(true).build(); storage.create(requesterPaysBucket); fillRequesterPaysFile(storage, SML_FILE, SML_SIZE); }
@Test public void testGetServiceAccount() { String projectId = remoteStorageHelper.getOptions().getProjectId(); ServiceAccount serviceAccount = storage.getServiceAccount(projectId); assertNotNull(serviceAccount); assertTrue(serviceAccount.getEmail().endsWith(SERVICE_ACCOUNT_EMAIL_SUFFIX)); } }
@Test public void testCreateFromStream() { RemoteStorageHelper helper = RemoteStorageHelper.create(PROJECT_ID, JSON_KEY_STREAM); StorageOptions options = helper.getOptions(); assertEquals(PROJECT_ID, options.getProjectId()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getConnectTimeout()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getReadTimeout()); assertEquals(10, options.getRetrySettings().getMaxAttempts()); assertEquals(Duration.ofMillis(30000), options.getRetrySettings().getMaxRetryDelay()); assertEquals(Duration.ofMillis(120000), options.getRetrySettings().getTotalTimeout()); assertEquals(Duration.ofMillis(250), options.getRetrySettings().getInitialRetryDelay()); } }
@Test public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName) .setDefaultKmsKeyName(kmsKeyOneResourcePath) .setLocation(KMS_KEY_RING_LOCATION) .build()); try { assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
private void retentionPolicyLockRequesterPays(boolean requesterPays) throws ExecutionException, InterruptedException { String projectId = remoteStorageHelper.getOptions().getProjectId(); String bucketName = RemoteStorageHelper.generateBucketName(); BucketInfo bucketInfo; if (requesterPays) { assertNotNull(remoteBucket.getRetentionEffectiveTime()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS);
@Test public void testGetBucketLifecycleRules() { String lifecycleTestBucketName = RemoteStorageHelper.generateBucketName(); storage.create( BucketInfo.newBuilder(lifecycleTestBucketName)
@Test public void testListBucketRequesterPaysFails() throws InterruptedException { String projectId = remoteStorageHelper.getOptions().getProjectId(); Iterator<Bucket> bucketIterator = storage .list( Storage.BucketListOption.prefix(BUCKET), Storage.BucketListOption.fields(), Storage.BucketListOption.userProject(projectId)) .iterateAll() .iterator(); while (!bucketIterator.hasNext()) { Thread.sleep(500); bucketIterator = storage .list(Storage.BucketListOption.prefix(BUCKET), Storage.BucketListOption.fields()) .iterateAll() .iterator(); } while (bucketIterator.hasNext()) { Bucket remoteBucket = bucketIterator.next(); assertTrue(remoteBucket.getName().startsWith(BUCKET)); assertNull(remoteBucket.getCreateTime()); assertNull(remoteBucket.getSelfLink()); } }
@Test public void testAttemptObjectDeleteWithRetentionPolicy() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName).setRetentionPeriod(RETENTION_PERIOD).build()); assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); String blobName = "test-create-with-retention-policy"; BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); Blob remoteBlob = storage.create(blobInfo); assertNotNull(remoteBlob.getRetentionExpirationTime()); try { remoteBlob.delete(); fail("Expected failure on delete from retentionPolicy"); } catch (StorageException ex) { // expected } finally { Thread.sleep(RETENTION_PERIOD_IN_MILLISECONDS); RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
@BeforeClass public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); storage.create( BucketInfo.newBuilder(BUCKET) .setLocation("us") .setLifecycleRules( ImmutableList.of( new LifecycleRule( LifecycleAction.newDeleteAction(), LifecycleCondition.newBuilder().setAge(1).build()))) .build()); // Prepare KMS KeyRing for CMEK tests prepareKmsKeys(); }
throws StorageHelperException { try { HttpTransportOptions transportOptions = StorageOptions.getDefaultHttpTransportOptions(); transportOptions = transportOptions.toBuilder().setConnectTimeout(60000).setReadTimeout(60000).build(); StorageOptions storageOptions = StorageOptions.newBuilder() .setCredentials(GoogleCredentials.fromStream(keyStream)) .setProjectId(projectId) .setRetrySettings(retrySettings()) .setTransportOptions(transportOptions) .build(); return new RemoteStorageHelper(storageOptions); } catch (IOException ex) { if (log.isLoggable(Level.WARNING)) {
String projectId = remoteStorageHelper.getOptions().getProjectId();
@AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { for (String bucket : new String[] {BUCKET, REQUESTER_PAYS_BUCKET}) { if (!RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS, project) && log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", bucket); } } } }
@Test public void testListBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS);
@BeforeClass public static void beforeClass() throws InterruptedException, TimeoutException { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); storage = storageHelper.getOptions().getService(); storage.create(BucketInfo.of(BUCKET)); storage.create( BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), CSV_CONTENT.getBytes(StandardCharsets.UTF_8)); storage.create( BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder( TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(TABLE_SCHEMA) .build(); Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); assertNull(job.getStatus().getError()); }
/** * Creates a {@code RemoteStorageHelper} object using default project id and authentication * credentials. */ public static RemoteStorageHelper create() throws StorageHelperException { HttpTransportOptions transportOptions = StorageOptions.getDefaultHttpTransportOptions(); transportOptions = transportOptions.toBuilder().setConnectTimeout(60000).setReadTimeout(60000).build(); StorageOptions storageOptions = StorageOptions.newBuilder() .setRetrySettings(retrySettings()) .setTransportOptions(transportOptions) .build(); return new RemoteStorageHelper(storageOptions); }
private static void prepareKmsKeys() throws IOException { String projectId = remoteStorageHelper.getOptions().getProjectId(); GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); ManagedChannel kmsChannel = ManagedChannelBuilder.forTarget("cloudkms.googleapis.com:443").build(); KeyManagementServiceBlockingStub kmsStub = KeyManagementServiceGrpc.newBlockingStub(kmsChannel) .withCallCredentials(MoreCallCredentials.from(credentials)); IAMPolicyGrpc.IAMPolicyBlockingStub iamStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) .withCallCredentials(MoreCallCredentials.from(credentials)); ensureKmsKeyRingExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); ensureKmsKeyRingIamPermissionsForTests( iamStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); kmsKeyOneResourcePath = ensureKmsKeyExistsForTests( kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_ONE_NAME); kmsKeyTwoResourcePath = ensureKmsKeyExistsForTests( kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_TWO_NAME); }