/** * Returns a {@code WriteChannel} object for writing to this blob. By default any md5 and crc32c * values in the current blob are ignored unless requested via the {@code * BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. * * <p>Example of writing the blob's content through a writer. * * <pre>{@code * byte[] content = "Hello, World!".getBytes(UTF_8); * try (WriteChannel writer = blob.writer()) { * try { * writer.write(ByteBuffer.wrap(content, 0, content.length)); * } catch (Exception ex) { * // handle exception * } * } * }</pre> * * @param options target blob options * @throws StorageException upon failure */ public WriteChannel writer(BlobWriteOption... options) { return storage.writer(this, options); }
@Override public void run() { try { if (directUpload) { byte[] content = Files.readAllBytes(localFile.toPath()); Blob result = mClient.create(sourceBlob, content); LOG.debug("Upload file {} to gs://{}/{}", localFile, gsBucket, gsKey); LOG.trace("Upload file {}, Blob: {}", result); } else { long startTime = System.nanoTime(); try (WriteChannel out = mClient.writer(sourceBlob); FileChannel in = new FileInputStream(localFile).getChannel(); ) { ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 1024 * 5); // 5 MiB buffer (remember this is pr. thread) int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { buffer.flip(); out.write(buffer); buffer.clear(); } } long elapsedTime = System.nanoTime() - startTime; LOG.debug("Upload file {} to gs://{}/{} in {} msec", localFile, gsBucket, gsKey, (elapsedTime / 1000000.0)); } } catch (IOException e) { throw new RuntimeException(e); } } });
/** Example of writing a blob's content through a writer. */ // [TARGET writer(BlobInfo, BlobWriteOption...)] // [VARIABLE "my_unique_bucket"] // [VARIABLE "my_blob_name"] public void writer(String bucketName, String blobName) throws IOException { // [START writer] BlobId blobId = BlobId.of(bucketName, blobName); byte[] content = "Hello, World!".getBytes(UTF_8); BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build(); try (WriteChannel writer = storage.writer(blobInfo)) { try { writer.write(ByteBuffer.wrap(content, 0, content.length)); } catch (Exception ex) { // handle exception } } // [END writer] }
WriteChannel writer = instance.writer(blobInfo);
@Test public void testWriter() throws Exception { initializeExpectedBlob(2); BlobWriteChannel channel = createMock(BlobWriteChannel.class); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.writer(eq(expectedBlob))).andReturn(channel); replay(storage); initializeBlob(); assertSame(channel, blob.writer()); }
storage.writer( infoBuilder.build(), writeOptions.toArray(new Storage.BlobWriteOption[writeOptions.size()])));
@Test public void testWriterWithKmsKeyName() throws Exception { initializeExpectedBlob(2); BlobWriteChannel channel = createMock(BlobWriteChannel.class); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.kmsKeyName(KMS_KEY_NAME)))) .andReturn(channel); replay(storage); initializeBlob(); assertSame(channel, blob.writer(BlobWriteOption.kmsKeyName(KMS_KEY_NAME))); }
@Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; int[] blobSizes = {0, 700, 1024 * 256, 2 * 1024 * 1024, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1}; Random rnd = new Random(); for (int blobSize : blobSizes) { String blobName = blobNamePrefix + blobSize; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] bytes = new byte[blobSize]; rnd.nextBytes(bytes); try (WriteChannel writer = storage.writer(blob)) { writer.write(ByteBuffer.wrap(bytes)); } ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ReadChannel reader = storage.reader(blob.getBlobId())) { ByteBuffer buffer = ByteBuffer.allocate(64 * 1024); while (reader.read(buffer) > 0) { buffer.flip(); output.write(buffer.array(), 0, buffer.limit()); buffer.clear(); } } assertArrayEquals(bytes, output.toByteArray()); assertTrue(storage.delete(BUCKET, blobName)); } }
@Test public void testReadAndWriteChannels() throws IOException { String blobName = "test-read-and-write-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId())) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); }
@Test public void testWriteChannelFail() throws IOException { String blobName = "test-write-channel-blob-fail"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName, -1L).build(); try { try (WriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.generationMatch())) { writer.write(ByteBuffer.allocate(42)); } fail("StorageException was expected"); } catch (StorageException ex) { // expected } }
@Test public void testWriterWithEncryptionKey() throws Exception { initializeExpectedBlob(2); BlobWriteChannel channel = createMock(BlobWriteChannel.class); expect(storage.getOptions()).andReturn(mockOptions); expect(storage.writer(eq(expectedBlob), eq(BlobWriteOption.encryptionKey(BASE64_KEY)))) .andReturn(channel) .times(2); replay(storage); initializeBlob(); assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(BASE64_KEY))); assertSame(channel, blob.writer(BlobWriteOption.encryptionKey(KEY))); }
@Test(timeout = 5000) public void testWriteChannelWithConnectionPool() throws IOException { TransportOptions transportOptions = HttpTransportOptions.newBuilder() .setHttpTransportFactory(new CustomHttpTransportFactory()) .build(); Storage storageWithPool = StorageOptions.newBuilder().setTransportOptions(transportOptions).build().getService(); String blobName = "test-custom-pool-management"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storageWithPool.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } try (WriteChannel writer = storageWithPool.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } }
@Test public void testReadAndWriteChannelWithEncryptionKey() throws IOException { String blobName = "test-read-write-channel-with-customer-key-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.encryptionKey(BASE64_KEY))) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId(), Storage.BlobSourceOption.decryptionKey(KEY))) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }
@Test public void testWriteChannelExistingBlob() throws IOException { String blobName = "test-write-channel-existing-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); storage.create(blob); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(stringBytes)); } assertArrayEquals(stringBytes, storage.readAllBytes(blob.getBlobId())); assertTrue(storage.delete(BUCKET, blobName)); }
@Test public void testWriterWithEncryptionKey() { BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); EasyMock.expect(storageRpcMock.open(info.toPb(), ENCRYPTION_KEY_OPTIONS)) .andReturn("upload-id") .times(2); EasyMock.replay(storageRpcMock); initializeService(); WriteChannel channel = storage.writer(info, BlobWriteOption.encryptionKey(KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); channel = storage.writer(info, BlobWriteOption.encryptionKey(BASE64_KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); }
@Test public void testWriterWithKmsKeyName() { BlobInfo info = BLOB_INFO1.toBuilder().setMd5(null).setCrc32c(null).build(); EasyMock.expect(storageRpcMock.open(info.toPb(), KMS_KEY_NAME_OPTIONS)) .andReturn("upload-id") .times(2); EasyMock.replay(storageRpcMock); initializeService(); WriteChannel channel = storage.writer(info, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); assertNotNull(channel); assertTrue(channel.isOpen()); channel = storage.writer(info, BlobWriteOption.kmsKeyName(KMS_KEY_NAME)); assertNotNull(channel); assertTrue(channel.isOpen()); }
assertEquals(chunkSize, numReadBytes); assertArrayEquals(Arrays.copyOf(content, chunkSize), readBytes.array()); try (WriteChannel writer = storage.writer(blob)) { byte[] newContent = new byte[blobSize]; random.nextBytes(newContent);
@Test public void testWriterWithOptions() { BlobInfo info = BLOB_INFO1.toBuilder().setMd5(CONTENT_MD5).setCrc32c(CONTENT_CRC32C).build(); EasyMock.expect(storageRpcMock.open(info.toPb(), BLOB_TARGET_OPTIONS_CREATE)) .andReturn("upload-id"); EasyMock.replay(storageRpcMock); initializeService(); WriteChannel channel = storage.writer( info, BLOB_WRITE_METAGENERATION, BLOB_WRITE_NOT_EXIST, BLOB_WRITE_PREDEFINED_ACL, BLOB_WRITE_CRC2C, BLOB_WRITE_MD5_HASH); assertNotNull(channel); assertTrue(channel.isOpen()); }
@Test public void testWriter() { BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); BlobInfo infoWithHashes = infoBuilder.setMd5(CONTENT_MD5).setCrc32c(CONTENT_CRC32C).build(); BlobInfo infoWithoutHashes = infoBuilder.setMd5(null).setCrc32c(null).build(); EasyMock.expect(storageRpcMock.open(infoWithoutHashes.toPb(), EMPTY_RPC_OPTIONS)) .andReturn("upload-id"); EasyMock.replay(storageRpcMock); initializeService(); WriteChannel channel = storage.writer(infoWithHashes); assertNotNull(channel); assertTrue(channel.isOpen()); }
@Test public void testReadAndWriteCaptureChannels() throws IOException { String blobName = "test-read-and-write-capture-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; WriteChannel writer = storage.writer(blob); stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); RestorableState<WriteChannel> writerState = writer.capture(); WriteChannel secondWriter = writerState.restore(); secondWriter.write(ByteBuffer.wrap(stringBytes)); secondWriter.close(); ByteBuffer readBytes; ByteBuffer readStringBytes; ReadChannel reader = storage.reader(blob.getBlobId()); reader.setChunkSize(BLOB_BYTE_CONTENT.length); readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); reader.read(readBytes); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel secondReader = readerState.restore(); readStringBytes = ByteBuffer.allocate(stringBytes.length); secondReader.read(readStringBytes); reader.close(); secondReader.close(); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }