/** * Deletes a bucket, even if non-empty. Objects in the bucket are listed and deleted until bucket * deletion succeeds or {@code timeout} expires. To allow for the timeout, this method uses a * separate thread to send the delete requests. Use {@link #forceDelete(Storage storage, String * bucket)} if spawning an additional thread is undesirable, such as in the App Engine production * runtime. * * @param storage the storage service to be used to issue requests * @param bucket the bucket to be deleted * @param timeout the maximum time to wait * @param unit the time unit of the timeout argument * @return true if deletion succeeded, false if timeout expired * @throws InterruptedException if the thread deleting the bucket is interrupted while waiting * @throws ExecutionException if an exception was thrown while deleting bucket or bucket objects */ public static Boolean forceDelete(Storage storage, String bucket, long timeout, TimeUnit unit) throws InterruptedException, ExecutionException { return forceDelete(storage, bucket, timeout, unit, ""); }
@AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { for (String bucket : new String[] {BUCKET, REQUESTER_PAYS_BUCKET}) { if (!RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS, project) && log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", bucket); } } } }
@AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (bigquery != null) { RemoteBigQueryHelper.forceDelete(bigquery, DATASET); } if (storage != null) { boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 10, TimeUnit.SECONDS); if (!wasDeleted && LOG.isLoggable(Level.WARNING)) { LOG.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); } } }
@AfterClass public static void afterClass() throws ExecutionException, InterruptedException { if (storage != null) { // In beforeClass, we make buckets auto-delete blobs older than a day old. // Here, delete all buckets older than 2 days. They should already be empty and easy. long cleanTime = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2); long cleanTimeout = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1); RemoteStorageHelper.cleanBuckets(storage, cleanTime, cleanTimeout); boolean wasDeleted = RemoteStorageHelper.forceDelete(storage, BUCKET, 1, TimeUnit.MINUTES); if (!wasDeleted && log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); } } }
@AfterClass public static void afterClass() throws Exception { topicAdminClient.close(); if (notificationService != null) { boolean wasDeleted = RemoteStorageHelper.forceDelete(storageService, BUCKET, 5, TimeUnit.SECONDS); if (!wasDeleted && log.isLoggable(Level.WARNING)) { log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); } } }
@Test public void testForceDeleteNoTimeout() { Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1); EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2); ArrayList<BlobId> ids = new ArrayList<>(); ids.add(BLOB_ID1); ids.add(BLOB_ID2); EasyMock.expect(storageMock.delete(ids)).andReturn(Collections.nCopies(2, true)).anyTimes(); EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) .andReturn(blobPage); EasyMock.expect(storageMock.delete(BUCKET_NAME)).andReturn(true); EasyMock.replay(storageMock, blob1, blob2); RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME); EasyMock.verify(storageMock); }
@Test public void testForceDelete() throws InterruptedException, ExecutionException { Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1); EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2); ArrayList<BlobId> ids = new ArrayList<>(); ids.add(BLOB_ID1); ids.add(BLOB_ID2); EasyMock.expect(storageMock.delete(ids)).andReturn(Collections.nCopies(2, true)); EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) .andReturn(blobPage); EasyMock.expect(storageMock.delete(BUCKET_NAME)).andReturn(true); EasyMock.replay(storageMock, blob1, blob2); assertTrue(RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME, 5, TimeUnit.SECONDS)); EasyMock.verify(storageMock, blob1, blob2); }
@Test public void testForceDeleteFail() throws InterruptedException, ExecutionException { Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1); EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2); ArrayList<BlobId> ids = new ArrayList<>(); ids.add(BLOB_ID1); ids.add(BLOB_ID2); EasyMock.expect(storageMock.delete(ids)).andReturn(Collections.nCopies(2, true)).anyTimes(); EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) .andReturn(blobPage); EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(FATAL_EXCEPTION); EasyMock.replay(storageMock, blob1, blob2); thrown.expect(ExecutionException.class); try { RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME, 5, TimeUnit.SECONDS); } finally { EasyMock.verify(storageMock); } }
@Test public void testForceDeleteNoTimeoutFail() { Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1); EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2); ArrayList<BlobId> ids = new ArrayList<>(); ids.add(BLOB_ID1); ids.add(BLOB_ID2); EasyMock.expect(storageMock.delete(ids)).andReturn(Collections.nCopies(2, true)).anyTimes(); EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) .andReturn(blobPage); EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(FATAL_EXCEPTION); EasyMock.replay(storageMock, blob1, blob2); thrown.expect(StorageException.class); try { RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME); } finally { EasyMock.verify(storageMock); } }
@Test public void testForceDeleteTimeout() throws InterruptedException, ExecutionException { Storage storageMock = EasyMock.createMock(Storage.class); EasyMock.expect(blob1.getBlobId()).andReturn(BLOB_ID1).anyTimes(); EasyMock.expect(blob2.getBlobId()).andReturn(BLOB_ID2).anyTimes(); ArrayList<BlobId> ids = new ArrayList<>(); ids.add(BLOB_ID1); ids.add(BLOB_ID2); EasyMock.expect(storageMock.delete(ids)).andReturn(Collections.nCopies(2, true)).anyTimes(); EasyMock.expect(storageMock.list(BUCKET_NAME, BlobListOption.versions(true))) .andReturn(blobPage) .anyTimes(); EasyMock.expect(storageMock.delete(BUCKET_NAME)).andThrow(RETRYABLE_EXCEPTION).anyTimes(); EasyMock.replay(storageMock, blob1, blob2); assertFalse( RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME, 50, TimeUnit.MICROSECONDS)); EasyMock.verify(storageMock); }
EasyMock.replay(storageMock, blob1, blob2); try { RemoteStorageHelper.forceDelete(storageMock, BUCKET_NAME, 5, TimeUnit.SECONDS, USER_PROJECT); } finally { EasyMock.verify(storageMock);
RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS);
@Override public void run() { Page<Bucket> buckets = storage.list(Storage.BucketListOption.prefix(BUCKET_NAME_PREFIX)); for (Bucket bucket : buckets.iterateAll()) { if (bucket.getCreateTime() < olderThan) { try { for (Blob blob : bucket .list( BlobListOption.fields( Storage.BlobField.EVENT_BASED_HOLD, Storage.BlobField.TEMPORARY_HOLD)) .iterateAll()) { if (blob.getEventBasedHold() == true || blob.getTemporaryHold() == true) { storage.update( blob.toBuilder() .setTemporaryHold(false) .setEventBasedHold(false) .build()); } } forceDelete(storage, bucket.getName()); } catch (Exception e) { // Ignore the exception, maybe the bucket is being deleted by someone else. } } } } };
@Test public void testAttemptObjectDeleteWithRetentionPolicy() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName).setRetentionPeriod(RETENTION_PERIOD).build()); assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); String blobName = "test-create-with-retention-policy"; BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); Blob remoteBlob = storage.create(blobInfo); assertNotNull(remoteBlob.getRetentionExpirationTime()); try { remoteBlob.delete(); fail("Expected failure on delete from retentionPolicy"); } catch (StorageException ex) { // expected } finally { Thread.sleep(RETENTION_PERIOD_IN_MILLISECONDS); RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
@Test public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName) .setDefaultKmsKeyName(kmsKeyOneResourcePath) .setLocation(KMS_KEY_RING_LOCATION) .build()); try { assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
@Test public void testUpdateBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName) .setDefaultKmsKeyName(kmsKeyOneResourcePath) .setLocation(KMS_KEY_RING_LOCATION) .build()); try { assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(kmsKeyTwoResourcePath).build().update(); assertEquals(kmsKeyTwoResourcePath, updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
@Test public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket bucket = storage.create( BucketInfo.newBuilder(bucketName) .setDefaultKmsKeyName(kmsKeyOneResourcePath) .setLocation(KMS_KEY_RING_LOCATION) .build()); assertEquals(bucket.getDefaultKmsKeyName(), kmsKeyOneResourcePath); try { String blobName = "test-create-with-default-kms-key-name-blob"; BlobInfo blob = BlobInfo.newBuilder(bucket, blobName).build(); Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); byte[] readBytes = storage.readAllBytes(bucketName, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
assertNotNull(remoteBucket.getRetentionEffectiveTime()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS);
@Test public void testEnableDisableBucketDefaultEventBasedHold() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(true).build()); try { assertTrue(remoteBucket.getDefaultEventBasedHold()); remoteBucket = storage.get( bucketName, Storage.BucketGetOption.fields(BucketField.DEFAULT_EVENT_BASED_HOLD)); assertTrue(remoteBucket.getDefaultEventBasedHold()); String blobName = "test-create-with-event-based-hold"; BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); Blob remoteBlob = storage.create(blobInfo); assertTrue(remoteBlob.getEventBasedHold()); remoteBlob = storage.get( blobInfo.getBlobId(), Storage.BlobGetOption.fields(BlobField.EVENT_BASED_HOLD)); assertTrue(remoteBlob.getEventBasedHold()); remoteBlob = remoteBlob.toBuilder().setEventBasedHold(false).build().update(); assertFalse(remoteBlob.getEventBasedHold()); remoteBucket = remoteBucket.toBuilder().setDefaultEventBasedHold(false).build().update(); assertFalse(remoteBucket.getDefaultEventBasedHold()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }
@Test public void testRetentionPolicyNoLock() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create( BucketInfo.newBuilder(bucketName).setRetentionPeriod(RETENTION_PERIOD).build()); try { assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); assertNotNull(remoteBucket.getRetentionEffectiveTime()); assertNull(remoteBucket.retentionPolicyIsLocked()); remoteBucket = storage.get(bucketName, Storage.BucketGetOption.fields(BucketField.RETENTION_POLICY)); assertEquals(RETENTION_PERIOD, remoteBucket.getRetentionPeriod()); assertNotNull(remoteBucket.getRetentionEffectiveTime()); assertNull(remoteBucket.retentionPolicyIsLocked()); String blobName = "test-create-with-retention-policy-hold"; BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); Blob remoteBlob = storage.create(blobInfo); assertNotNull(remoteBlob.getRetentionExpirationTime()); remoteBucket = remoteBucket.toBuilder().setRetentionPeriod(null).build().update(); assertNull(remoteBucket.getRetentionPeriod()); remoteBucket = remoteBucket.toBuilder().setRetentionPeriod(null).build().update(); assertNull(remoteBucket.getRetentionPeriod()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } }