private void prepareUploadPart(String container, String key, String uploadId, int part, Payload payload, long offset, long size, SortedMap<Integer, String> etags) { Payload chunkedPart = slicer.slice(payload, offset, size); String eTag = null; try { eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } catch (KeyNotFoundException e) { // note that because of eventual consistency, the upload id may not be // present yet we may wish to add this condition to the retry handler // we may also choose to implement ListParts and wait for the uploadId // to become available there. eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } } }
private void prepareUploadPart(String container, String key, String uploadId, int part, Payload payload, long offset, long size, SortedMap<Integer, String> etags) { Payload chunkedPart = slicer.slice(payload, offset, size); String eTag = null; try { eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } catch (KeyNotFoundException e) { // note that because of eventual consistency, the upload id may not be // present yet we may wish to add this condition to the retry handler // we may also choose to implement ListParts and wait for the uploadId // to become available there. eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } } }
private void prepareUploadPart(String container, String key, String uploadId, int part, Payload payload, long offset, long size, SortedMap<Integer, String> etags) { Payload chunkedPart = slicer.slice(payload, offset, size); String eTag = null; try { eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } catch (KeyNotFoundException e) { // note that because of eventual consistency, the upload id may not be // present yet we may wish to add this condition to the retry handler // we may also choose to implement ListParts and wait for the uploadId // to become available there. eTag = client.uploadPart(container, key, part, uploadId, chunkedPart); etags.put(Integer.valueOf(part), eTag); } } }
@Override public String execute(String container, Blob blob) { String blobName = blob.getMetadata().getName(); Payload payload = blob.getPayload(); Long length = payload.getContentMetadata().getContentLength(); checkNotNull(length, "please invoke payload.getContentMetadata().setContentLength(length) prior to azure block upload"); checkArgument(length <= (MAX_NUMBER_OF_BLOCKS * MAX_BLOCK_SIZE)); long offset = 0L; List<String> blockIds = Lists.newArrayList(); long bytesWritten = 0; while (offset < length) { long chunkSize = MAX_BLOCK_SIZE; if (length - offset < MAX_BLOCK_SIZE) { chunkSize = length % MAX_BLOCK_SIZE; } bytesWritten += chunkSize; Payload block = slicer.slice(payload, offset, chunkSize); offset += MultipartUploadStrategy.MAX_BLOCK_SIZE; String blockName = blobName + "-" + offset + "-" + new SecureRandom().nextInt(); byte blockIdBytes[] = Hashing.md5().hashBytes(blockName.getBytes()).asBytes(); String blockId = BaseEncoding.base64().encode(blockIdBytes); blockIds.add(blockId); client.putBlock(container, blobName, blockId, block); } checkState(bytesWritten == length, "Wrote %s bytes, but we wanted to write %s bytes", bytesWritten, length); return client.putBlockList(container, blobName, blockIds); } }
@Test public void testIterableSliceExpectedSingle() throws IOException { PayloadSlicer slicer = new BasePayloadSlicer(); String contents = "aaaaaaaaaabbbbbbbbbbccccc"; Payload payload = new InputStreamPayload(new ByteArrayInputStream(contents.getBytes(Charsets.US_ASCII))); Iterator<Payload> iter = slicer.slice(payload, 25).iterator(); assertTrue(iter.hasNext(), "Not enough results"); assertEquals(Strings2.toStringAndClose(iter.next().openStream()), contents); assertFalse(iter.hasNext()); }
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s with uploadId %s", part, key, container, uploadId)); final long start = System.currentTimeMillis();
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s with uploadId %s", part, key, container, uploadId)); final long start = System.currentTimeMillis();
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s with uploadId %s", part, key, container, uploadId)); final long start = System.currentTimeMillis();
@Test public void testIterableSliceExpectedMulti() throws IOException { PayloadSlicer slicer = new BasePayloadSlicer(); Payload payload = new InputStreamPayload(new ByteArrayInputStream("aaaaaaaaaabbbbbbbbbbccccc".getBytes(Charsets.US_ASCII))); Iterator<Payload> iter = slicer.slice(payload, 10).iterator(); assertTrue(iter.hasNext(), "Not enough results"); assertEquals(Strings2.toStringAndClose(iter.next().getInput()), "aaaaaaaaaa"); assertTrue(iter.hasNext(), "Not enough results"); assertEquals(Strings2.toStringAndClose(iter.next().getInput()), "bbbbbbbbbb"); assertTrue(iter.hasNext(), "Not enough results"); assertEquals(Strings2.toStringAndClose(iter.next().getInput()), "ccccc"); assertFalse(iter.hasNext()); }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
@Test public void testIterableSliceWithRepeatingByteSourceLargerPartSize() throws IOException { PayloadSlicer slicer = new BasePayloadSlicer(); ByteSource byteSource = ByteSource.wrap("aaaaaaaaaabbbbbbbbbbccccc".getBytes(Charsets.UTF_8)); /* 25 chars */ Payload payload = new ByteSourcePayload(byteSource); Iterator<Payload> iter = slicer.slice(payload, 50).iterator(); Payload part; assertTrue(iter.hasNext(), "Not enough results"); part = iter.next(); assertEquals(Strings2.toStringAndClose(part.getInput()), "aaaaaaaaaabbbbbbbbbbccccc"); assertEquals(part.getContentMetadata().getContentLength(), Long.valueOf(25)); assertFalse(iter.hasNext()); } }
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); final long start = System.currentTimeMillis();
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); final long start = System.currentTimeMillis();
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { abortMultipartUpload(mpu); throw re; } }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { abortMultipartUpload(mpu); throw re; } }
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); final long start = System.currentTimeMillis();
Payload chunkedPart = slicer.slice(payload, offset, size); logger.debug(String.format("async uploading part %s of %s to container %s", part, key, container)); final long start = System.currentTimeMillis();
@Test public void testIterableSliceWithRepeatingByteSourceSmallerPartSize() throws IOException { PayloadSlicer slicer = new BasePayloadSlicer(); ByteSource byteSource = ByteSource.wrap("aaaaaaaaaabbbbbbbbbbccccc".getBytes(Charsets.UTF_8)); /* 25 chars */ Payload payload = new ByteSourcePayload(byteSource); Iterator<Payload> iter = slicer.slice(payload, 10).iterator(); Payload part; assertTrue(iter.hasNext(), "Not enough results"); part = iter.next(); assertEquals(Strings2.toStringAndClose(part.getInput()), "aaaaaaaaaa"); assertEquals(part.getContentMetadata().getContentLength(), Long.valueOf(10)); assertTrue(iter.hasNext(), "Not enough results"); part = iter.next(); assertEquals(Strings2.toStringAndClose(part.getInput()), "bbbbbbbbbb"); assertEquals(part.getContentMetadata().getContentLength(), Long.valueOf(10)); assertTrue(iter.hasNext(), "Not enough results"); part = iter.next(); assertEquals(Strings2.toStringAndClose(part.getInput()), "ccccc"); assertEquals(part.getContentMetadata().getContentLength(), Long.valueOf(5)); assertFalse(iter.hasNext()); }