@Override public int write(ByteBuffer src) throws IOException { synchronized (this) { checkOpen(); int amt = channel.write(src); if (amt > 0) { position += amt; size += amt; } return amt; } }
/** Example of writing the blob's content through a writer. */ // [TARGET writer(BlobWriteOption...)] public void writer() throws IOException { // [START writer] byte[] content = "Hello, World!".getBytes(UTF_8); try (WriteChannel writer = blob.writer()) { try { writer.write(ByteBuffer.wrap(content, 0, content.length)); } catch (Exception ex) { // handle exception } } // [END writer] }
@Override public void run() { try { if (directUpload) { byte[] content = Files.readAllBytes(localFile.toPath()); Blob result = mClient.create(sourceBlob, content); LOG.debug("Upload file {} to gs://{}/{}", localFile, gsBucket, gsKey); LOG.trace("Upload file {}, Blob: {}", result); } else { long startTime = System.nanoTime(); try (WriteChannel out = mClient.writer(sourceBlob); FileChannel in = new FileInputStream(localFile).getChannel(); ) { ByteBuffer buffer = ByteBuffer.allocateDirect(1024 * 1024 * 5); // 5 MiB buffer (remember this is pr. thread) int bytesRead; while ((bytesRead = in.read(buffer)) > 0) { buffer.flip(); out.write(buffer); buffer.clear(); } } long elapsedTime = System.nanoTime() - startTime; LOG.debug("Upload file {} to gs://{}/{} in {} msec", localFile, gsBucket, gsKey, (elapsedTime / 1000000.0)); } } catch (IOException e) { throw new RuntimeException(e); } } });
/** Example of writing a blob's content through a writer. */ // [TARGET writer(BlobInfo, BlobWriteOption...)] // [VARIABLE "my_unique_bucket"] // [VARIABLE "my_blob_name"] public void writer(String bucketName, String blobName) throws IOException { // [START writer] BlobId blobId = BlobId.of(bucketName, blobName); byte[] content = "Hello, World!".getBytes(UTF_8); BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build(); try (WriteChannel writer = storage.writer(blobInfo)) { try { writer.write(ByteBuffer.wrap(content, 0, content.length)); } catch (Exception ex) { // handle exception } } // [END writer] }
@Test public void testWrite() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1); buffer.put((byte) 'B'); assertThat(chan.position()).isEqualTo(0L); assertThat(chan.size()).isEqualTo(0L); when(gcsChannel.write(eq(buffer))).thenReturn(1); assertThat(chan.write(buffer)).isEqualTo(1); assertThat(chan.position()).isEqualTo(1L); assertThat(chan.size()).isEqualTo(1L); verify(gcsChannel).write(any(ByteBuffer.class)); verify(gcsChannel, times(5)).isOpen(); verifyNoMoreInteractions(gcsChannel); }
@Test public void testReadAndWriteChannels() throws IOException { String blobName = "test-read-and-write-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId())) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); }
@Test(timeout = 5000) public void testWriteChannelWithConnectionPool() throws IOException { TransportOptions transportOptions = HttpTransportOptions.newBuilder() .setHttpTransportFactory(new CustomHttpTransportFactory()) .build(); Storage storageWithPool = StorageOptions.newBuilder().setTransportOptions(transportOptions).build().getService(); String blobName = "test-custom-pool-management"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storageWithPool.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } try (WriteChannel writer = storageWithPool.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } }
@Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; int[] blobSizes = {0, 700, 1024 * 256, 2 * 1024 * 1024, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1}; Random rnd = new Random(); for (int blobSize : blobSizes) { String blobName = blobNamePrefix + blobSize; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] bytes = new byte[blobSize]; rnd.nextBytes(bytes); try (WriteChannel writer = storage.writer(blob)) { writer.write(ByteBuffer.wrap(bytes)); } ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ReadChannel reader = storage.reader(blob.getBlobId())) { ByteBuffer buffer = ByteBuffer.allocate(64 * 1024); while (reader.read(buffer) > 0) { buffer.flip(); output.write(buffer.array(), 0, buffer.limit()); buffer.clear(); } } assertArrayEquals(bytes, output.toByteArray()); assertTrue(storage.delete(BUCKET, blobName)); } }
@Test public void testReadAndWriteChannelWithEncryptionKey() throws IOException { String blobName = "test-read-write-channel-with-customer-key-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.encryptionKey(BASE64_KEY))) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId(), Storage.BlobSourceOption.decryptionKey(KEY))) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }
@Test public void testWriteChannelFail() throws IOException { String blobName = "test-write-channel-blob-fail"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName, -1L).build(); try { try (WriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.generationMatch())) { writer.write(ByteBuffer.allocate(42)); } fail("StorageException was expected"); } catch (StorageException ex) { // expected } }
@Test public void testWriteChannelExistingBlob() throws IOException { String blobName = "test-write-channel-existing-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); storage.create(blob); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(stringBytes)); } assertArrayEquals(stringBytes, storage.readAllBytes(blob.getBlobId())); assertTrue(storage.delete(BUCKET, blobName)); }
@Test public void testReadAndWriteCaptureChannels() throws IOException { String blobName = "test-read-and-write-capture-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; WriteChannel writer = storage.writer(blob); stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); RestorableState<WriteChannel> writerState = writer.capture(); WriteChannel secondWriter = writerState.restore(); secondWriter.write(ByteBuffer.wrap(stringBytes)); secondWriter.close(); ByteBuffer readBytes; ByteBuffer readStringBytes; ReadChannel reader = storage.reader(blob.getBlobId()); reader.setChunkSize(BLOB_BYTE_CONTENT.length); readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); reader.read(readBytes); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel secondReader = readerState.restore(); readStringBytes = ByteBuffer.allocate(stringBytes.length); secondReader.read(readStringBytes); reader.close(); secondReader.close(); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }
byte[] newContent = new byte[blobSize]; random.nextBytes(newContent); int numWrittenBytes = writer.write(ByteBuffer.wrap(newContent)); assertEquals(blobSize, numWrittenBytes);
@Test public void testSaveAndRestore() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); Capture<byte[]> capturedBuffer = Capture.newInstance(CaptureType.ALL); Capture<Long> capturedPosition = Capture.newInstance(CaptureType.ALL); storageRpcMock.write( eq(UPLOAD_ID), capture(capturedBuffer), eq(0), captureLong(capturedPosition), eq(DEFAULT_CHUNK_SIZE), eq(false)); expectLastCall().times(2); replay(storageRpcMock); ByteBuffer buffer1 = randomBuffer(DEFAULT_CHUNK_SIZE); ByteBuffer buffer2 = randomBuffer(DEFAULT_CHUNK_SIZE); writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); assertEquals(DEFAULT_CHUNK_SIZE, writer.write(buffer1)); assertArrayEquals(buffer1.array(), capturedBuffer.getValues().get(0)); assertEquals(new Long(0L), capturedPosition.getValues().get(0)); RestorableState<WriteChannel> writerState = writer.capture(); WriteChannel restoredWriter = writerState.restore(); assertEquals(DEFAULT_CHUNK_SIZE, restoredWriter.write(buffer2)); assertArrayEquals(buffer2.array(), capturedBuffer.getValues().get(1)); assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1)); }
RestorableState<WriteChannel> writerState = writer.capture(); WriteChannel restoredWriter = writerState.restore(); assertEquals(DEFAULT_CHUNK_SIZE, restoredWriter.write(buffer2)); assertArrayEquals(buffer2.array(), capturedBuffer.getValues().get(1)); assertEquals(new Long(DEFAULT_CHUNK_SIZE), capturedPosition.getValues().get(1));
@Override public void write(byte[] b) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(b); writer.write(buffer); }
@Override public void write(byte[] b, int off, int len) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(b, off, len); writer.write(buffer); }
@Override public void write(InputStream inputStream, String destination) throws IOException { String[] tokens = getBucketAndObjectFromPath(destination); Assert.state(tokens.length == 2, "Can only write to files, not buckets."); BlobInfo gcsBlobInfo = BlobInfo.newBuilder(BlobId.of(tokens[0], tokens[1])).build(); try (InputStream is = inputStream) { try (WriteChannel channel = this.gcs.writer(gcsBlobInfo)) { channel.write(ByteBuffer.wrap(StreamUtils.copyToByteArray(is))); } } }
@Override public void store(Value v) throws IOException { if (!v._key.home()) return; final byte payload[] = v.memOrLoad(); final GcsBlob blob = GcsBlob.of(v._key); Log.debug("Storing: " + blob.toString()); final ByteBuffer buffer = ByteBuffer.wrap(payload); storageProvider.getStorage().create(blob.getBlobInfo()).writer().write(buffer); }