private IOException asIoException(StorageException oops) { // RPC API can only throw StorageException, but CloudStorageFileSystemProvider // can only throw IOException. Square peg, round hole. // TODO(#810): Research if other codes should be translated similarly. if (oops.getCode() == 404) { return new NoSuchFileException(oops.getReason()); } Throwable cause = oops.getCause(); try { if (cause instanceof FileAlreadyExistsException) { throw new FileAlreadyExistsException(((FileAlreadyExistsException) cause).getReason()); } // fallback if (cause != null && cause instanceof IOException) { return (IOException) cause; } } catch (IOException okEx) { return okEx; } return new IOException(oops.getMessage(), oops); } }
private static StorageException translate(GoogleJsonError exception) { return new StorageException(exception); }
/** * @param exs StorageException to test * @return true if exs is a retryable error, otherwise false */ @VisibleForTesting boolean isRetryable(final StorageException exs) { if (exs.isRetryable()) { return true; } for (int code : config.retryableHttpCodes()) { if (exs.getCode() == code) { return true; } } return false; }
/** * Records a reopen attempt for the given StorageException, sleeping for an amount of time * dependent on the attempt number. Throws a StorageException if we've exhausted all reopens. * * @param exs The StorageException error that prompted this reopen attempt. */ private void handleReopenForStorageException(final StorageException exs) throws StorageException { reopens++; if (reopens > maxReopens) { throw new StorageException( exs.getCode(), "All " + maxReopens + " reopens failed. Waited a total of " + totalWaitTime + " ms between attempts", exs); } sleepForAttempt(reopens); }
@Override public Acl getAcl(final BlobId blob, final Entity entity) { try { ObjectAccessControl answer = runWithRetries( new Callable<ObjectAccessControl>() { @Override public ObjectAccessControl call() { return storageRpc.getAcl( blob.getBucket(), blob.getName(), blob.getGeneration(), entity.toPb()); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); return answer == null ? null : Acl.fromPb(answer); } catch (RetryHelperException e) { throw StorageException.translateAndThrow(e); } }
@Override public List<Acl> listAcls(final BlobId blob) { try { List<ObjectAccessControl> answer = runWithRetries( new Callable<List<ObjectAccessControl>>() { @Override public List<ObjectAccessControl> call() { return storageRpc.listAcls( blob.getBucket(), blob.getName(), blob.getGeneration()); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); return Lists.transform(answer, Acl.FROM_OBJECT_PB_FUNCTION); } catch (RetryHelperException e) { throw StorageException.translateAndThrow(e); } }
@Test public void testRetryableException() { BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); EasyMock.expect(storageRpcMock.get(blob.toPb(), EMPTY_RPC_OPTIONS)) .andThrow(new StorageException(500, "internalError")) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(storageRpcMock); storage = options .toBuilder() .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) .build() .getService(); initializeServiceDependentObjects(); Blob readBlob = storage.get(blob); assertEquals(expectedBlob1, readBlob); }
@Override public Tuple<String, byte[]> read( StorageObject from, Map<Option, ?> options, long position, int bytes) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_READ); Scope scope = tracer.withSpan(span); try { Get req = storage .objects() .get(from.getBucket(), from.getName()) .setGeneration(from.getGeneration()) HttpHeaders requestHeaders = req.getRequestHeaders(); requestHeaders.setRange(range.toString()); setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, options); ByteArrayOutputStream output = new ByteArrayOutputStream(bytes); HttpResponse httpResponse = req.executeMedia(); IOUtils.copy(lowLevelHttpResponse.getContent(), output); } catch (IllegalAccessException | NoSuchFieldException ex) { throw new StorageException( BaseServiceException.UNKNOWN_CODE, "Error parsing gzip response", ex); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE) { return Tuple.of(null, new byte[0]);
@Override public boolean delete(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_DELETE_BUCKET); Scope scope = tracer.withSpan(span); try { storage .buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return true; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return false; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public void process(InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { final BlobId id = BlobId.of(bucket, key); final BlobInfo.Builder blobInfoBuilder = BlobInfo.newBuilder(id); final List<Storage.BlobWriteOption> blobWriteOptions = new ArrayList<>(); new Object[]{ffFilename, bucket, key, e.getMessage()}, e); throw (e);
@Test public void testStorageException() { StorageException exception = new StorageException(500, "message"); assertEquals(500, exception.getCode()); assertEquals("message", exception.getMessage()); assertNull(exception.getReason()); assertTrue(exception.isRetryable()); exception = new StorageException(502, "message"); assertEquals(502, exception.getCode()); assertEquals("message", exception.getMessage()); assertNull(exception.getReason()); assertTrue(exception.isRetryable()); exception = new StorageException(503, "message"); assertEquals(503, exception.getCode()); assertEquals("message", exception.getMessage()); assertNull(exception.getReason()); assertTrue(exception.isRetryable()); exception = new StorageException(504, "message"); assertEquals(504, exception.getCode()); assertEquals("message", exception.getMessage()); assertNull(exception.getReason()); assertTrue(exception.isRetryable()); exception = new StorageException(429, "message"); assertEquals(429, exception.getCode()); assertEquals("message", exception.getMessage()); assertNull(exception.getReason());
@Test public void testCreateBlobFromStreamRetryableException() throws IOException { Capture<ByteArrayInputStream> capturedStream = Capture.newInstance(); ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT); BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); BlobInfo infoWithHashes = infoBuilder.setMd5(CONTENT_MD5).setCrc32c(CONTENT_CRC32C).build(); BlobInfo infoWithoutHashes = infoBuilder.setMd5(null).setCrc32c(null).build(); EasyMock.expect( storageRpcMock.create( EasyMock.eq(infoWithoutHashes.toPb()), EasyMock.capture(capturedStream), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new StorageException(500, "internalError")) .once(); EasyMock.replay(storageRpcMock); storage = options .toBuilder() .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) .build() .getService(); // Even though this exception is retryable, storage.create(BlobInfo, InputStream) // shouldn't retry. thrown.expect(StorageException.class); thrown.expectMessage("internalError"); storage.create(infoWithHashes, fileStream); }
final BlobId blobId = BlobId.of(bucketName, key, generation); throw new StorageException(404, "Blob " + blobId + " not found"); getLogger().error(e.getMessage(), e); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE);
@Test public void testBlobAcl() { BlobId blobId = BlobId.of(BUCKET, "test-blob-acl"); BlobInfo blob = BlobInfo.newBuilder(blobId).build(); storage.create(blob); assertNull(storage.getAcl(blobId, User.ofAllAuthenticatedUsers())); Acl acl = Acl.of(User.ofAllAuthenticatedUsers(), Role.READER); assertNotNull(storage.createAcl(blobId, acl)); Acl updatedAcl = storage.updateAcl(blobId, acl.toBuilder().setRole(Role.OWNER).build()); assertEquals(Role.OWNER, updatedAcl.getRole()); Set<Acl> acls = new HashSet<>(storage.listAcls(blobId)); assertTrue(acls.contains(updatedAcl)); assertNull(storage.getAcl(blobId, User.ofAllAuthenticatedUsers())); BlobId otherBlobId = BlobId.of(BUCKET, "test-blob-acl", -1L); try { assertNull(storage.getAcl(otherBlobId, User.ofAllAuthenticatedUsers())); fail("Expected an 'Invalid argument' exception"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Invalid argument"); fail("Expected an 'Invalid argument' exception"); } catch (StorageException e) { assertThat(e.getMessage()).contains("Invalid argument");
@Test public void testReadGenerationChanged() throws IOException { BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME); reader = new BlobReadChannel(options, blobId, EMPTY_RPC_OPTIONS); byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag1", firstResult)); expect( storageRpcMock.read( blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag2", secondResult)); replay(storageRpcMock); reader.read(firstReadBuffer); try { reader.read(secondReadBuffer); fail("Expected ReadChannel read to throw StorageException"); } catch (StorageException ex) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append("Blob ").append(blobId).append(" was updated while reading"); assertEquals(messageBuilder.toString(), ex.getMessage()); } }
@Override public ObjectAccessControl getAcl(String bucket, String object, Long generation, String entity) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_OBJECT_ACL); Scope scope = tracer.withSpan(span); try { return storage .objectAccessControls() .get(bucket, object, entity) .setGeneration(generation) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }
/** * @param bucketName the name of the bucket to check * @return whether requester pays is enabled for that bucket */ public boolean requesterPays(String bucketName) { initStorage(); try { // instead of true/false, this method returns true/null. Boolean isRP = storage.get(bucketName).requesterPays(); return isRP != null && isRP.booleanValue(); } catch (StorageException ex) { if (ex.getCode() == 400 && ex.getMessage().contains("Bucket is requester pays")) { return true; } throw ex; } }
@Override public ObjectAccessControl getDefaultAcl(String bucket, String entity) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_OBJECT_DEFAULT_ACL); Scope scope = tracer.withSpan(span); try { return storage.defaultObjectAccessControls().get(bucket, entity).execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public BucketAccessControl getAcl(String bucket, String entity, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_BUCKET_ACL); Scope scope = tracer.withSpan(span); try { return storage .bucketAccessControls() .get(bucket, entity) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }