protected long getNextChunkOffset() { long next = chunkOffset; chunkOffset += getChunkSize(); return next; }
/** * Phase 1 of the algorithm. * ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE} * until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts. */ @Test public void testWhenChunkSizeHasToStartGrowing() { MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly defaultPartSize chunkSize long length = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE; long chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1); assertEquals(strategy.getRemaining(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); // then chunkSize is increasing length += 1; chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * 2); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE / 2); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; while (partNumber <= algorithm.getParts()) { Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize); BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); algorithm.addCopied(partSize); if (algorithm.getRemaining() != 0) { Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining()); BlobUploader b = new BlobUploader(mpu, partNumber, payload);
/** * Phase 2 of the algorithm. * The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} * until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}. */ @Test public void testWhenPartsHasToStartGrowingFromMagnitudeBase() { MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining) long length = MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE; long chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1); assertEquals(strategy.getRemaining(), MAX_PART_SIZE); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); // then the number of parts is increasing length += 1; chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; while (partNumber <= algorithm.getParts()) { Payload slice = slicer.slice(payload, algorithm.getCopied(), partSize); BlobUploader b = new BlobUploader(mpu, partNumber++, slice); parts.add(repeatable ? executor.submit(b) : Futures.immediateFuture(b.call())); algorithm.addCopied(partSize); if (algorithm.getRemaining() != 0) { Payload slice = slicer.slice(payload, algorithm.getCopied(), algorithm.getRemaining()); BlobUploader b = new BlobUploader(mpu, partNumber, slice); parts.add(repeatable ? executor.submit(b) : Futures.immediateFuture(b.call()));
/** * Phase 3 of the algorithm. * The number of parts are increasing until {@code MAX_NUMBER_OF_PARTS} * while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}. */ @Test public void testWhenPartsExceedsMaxNumberOfParts() { MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); // upper limit while we still have exactly MAX_NUMBER_OF_PARTS parts (together with the remaining) long length = MAX_PART_SIZE * MAX_NUMBER_OF_PARTS; long chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS - 1); assertEquals(strategy.getRemaining(), MAX_PART_SIZE); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); // then the number of parts is increasing length += 1; chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MAX_PART_SIZE); assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); } }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(), "must provide content-length to use multi-part upload"); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides); int partNumber = 0; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); }
protected long getNextChunkOffset() { long next = chunkOffset; chunkOffset += getChunkSize(); return next; }
MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm( MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS); long chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), 0); assertEquals(strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), 0); assertEquals(strategy.getRemaining(), length); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length); chunkSize = strategy.calculateChunkSize(length); assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE); assertEquals(strategy.getParts(), 1); assertEquals(strategy.getRemaining(), 1); assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { abortMultipartUpload(mpu); throw re; } }
protected long getNextChunkOffset() { long next = chunkOffset; chunkOffset += getChunkSize(); return next; }
@Beta protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) { ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>(); MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides); try { long contentLength = blob.getMetadata().getContentMetadata().getContentLength(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts()); long partSize = algorithm.calculateChunkSize(contentLength); int partNumber = 1; for (Payload payload : slicer.slice(blob.getPayload(), partSize)) { BlobUploader b = new BlobUploader(mpu, partNumber++, payload); parts.add(executor.submit(b)); } return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts))); } catch (RuntimeException re) { abortMultipartUpload(mpu); throw re; } }
protected long getNextChunkOffset() { long next = chunkOffset; chunkOffset += getChunkSize(); return next; }
@Test(groups = { "integration", "live" }) public void testPutMultipartByteSource() throws Exception { long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1); BlobStore blobStore = view.getBlobStore(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(), blobStore.getMaximumNumberOfParts()); // make sure that we are creating multiple parts assertThat(algorithm.calculateChunkSize(length)).isLessThan(length); ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new ByteSourcePayload(byteSource); HashCode hashCode = byteSource.hash(Hashing.md5()); testPut(payload, hashCode, payload, length, new PutOptions().multipart(true)); }
@Test(groups = { "integration", "live" }) public void testPutMultipartInputStream() throws Exception { long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1); BlobStore blobStore = view.getBlobStore(); MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm( blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(), blobStore.getMaximumNumberOfParts()); // make sure that we are creating multiple parts assertThat(algorithm.calculateChunkSize(length)).isLessThan(length); ByteSource byteSource = TestUtils.randomByteSource().slice(0, length); Payload payload = new InputStreamPayload(byteSource.openStream()); testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true)); }