Refine search
om.setContentType( mimeType ); PutObjectResult result = null; result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om ); InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest ); .withPartSize(partSize) .withLastPart(isLastPart); partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() ); partNumber++; new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(), partETags ); CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() ); em.update( entity );
"The request parameter must be specified when completing a multipart upload"); String bucketName = completeMultipartUploadRequest.getBucketName(); String key = completeMultipartUploadRequest.getKey(); String uploadId = completeMultipartUploadRequest.getUploadId(); rejectNull(bucketName, "The bucket name parameter must be specified when completing a multipart upload"); rejectNull(uploadId, "The upload ID parameter must be specified when completing a multipart upload"); rejectNull(completeMultipartUploadRequest.getPartETags(), "The part ETags parameter must be specified when completing a multipart upload"); request.addParameter("uploadId", uploadId); populateRequesterPaysHeader(request, completeMultipartUploadRequest.isRequesterPays()); byte[] xml = RequestXmlFactory.convertToXmlByteArray(completeMultipartUploadRequest.getPartETags()); request.addHeader("Content-Type", "application/xml"); request.addHeader("Content-Length", String.valueOf(xml.length));
partETags.add(s3.uploadPart(uploadPartRequest).getPartETag()); new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), multipartUploadId, partETags) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) CompleteMultipartUploadResult res = s3.completeMultipartUpload(req); uploadResult.setBucketName(res.getBucketName()); uploadResult.setKey(res.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); return uploadResult;
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { monitor.uploadFailure(); publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
/** * Completes the multipart upload and returns the result. */ private UploadResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), uploadId, collectPartETags())); uploadComplete(); UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult; }
initRequest.setStorageClass(maybeStorageClass.get()); InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); .withFile(file) .withPartSize(partSize); partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); filePosition += partSize; CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(bucketName, key, initResponse.getUploadId(), partETags); s3Client.completeMultipartUpload(completeRequest); } catch (Exception e) { s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, initResponse.getUploadId()));
/** * Completes multipart upload. */ private void completeMultiPartUpload() throws IOException { AmazonClientException lastException; CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(mBucketName, mKey, mUploadId, mTags); do { try { mClient.completeMultipartUpload(completeRequest); LOG.debug("Completed multipart upload for key {} and id '{}' with {} partitions.", mKey, mUploadId, mTags.size()); return; } catch (AmazonClientException e) { lastException = e; } } while (mRetryPolicy.attempt()); // This point is only reached if the operation failed more // than the allowed retry count throw new IOException("Unable to complete multipart upload with id '" + mUploadId + "' to " + mKey, lastException); }
final InitiateMultipartUploadResult initResponse = getEndpoint().getS3Client().initiateMultipartUpload(initRequest); final long contentLength = objectMetadata.getContentLength(); final List<PartETag> partETags = new ArrayList<PartETag>(); partETags.add(getEndpoint().getS3Client().uploadPart(uploadRequest).getPartETag()); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(getConfiguration().getBucketName(), keyName, initResponse.getUploadId(), partETags); uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest); message.setHeader(S3Constants.E_TAG, uploadResult.getETag()); if (uploadResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
private void completeUpload(InitiateMultipartUploadResult initResponse, List<PartETag> eTags) throws Exception { CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(initResponse.getBucketName(), initResponse.getKey(), initResponse.getUploadId(), eTags); s3Client.completeMultipartUpload(completeRequest); }
amazonS3.initiateMultipartUpload(initiateUploadRequest); final String uploadId = initiateUploadResult.getUploadId(); copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey()); CopyPartResult copyPartResult = amazonS3.copyPart(copyPartRequest); eTags.add(copyPartResult.getPartETag()); } else { copyPartRequest.setDestinationSSECustomerKey(options.getSSECustomerKey()); CopyPartResult copyPartResult = amazonS3.copyPart(copyPartRequest); eTags.add(copyPartResult.getPartETag()); new CompleteMultipartUploadRequest() .withBucketName(destinationPath.getBucket()) .withKey(destinationPath.getKey()) .withUploadId(uploadId) .withPartETags(eTags); return amazonS3.completeMultipartUpload(completeUploadRequest);
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( bucket, key, currentState.getUploadId(), currentState.getPartETags()); try { s3.completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[]{ffFilename, completeResult.getETag(), currentState.getUploadId()}); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); if (completeResult.getETag() != null) {
/** * Send the CompleteMultipartUploadRequest to S3 if all the blocks of a file are uploaded into S3. * @param keyName file to upload into S3 */ private void verifyAndEmitFileMerge(String keyName) { if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) { return; } S3InitiateFileUploadOperator.UploadFileMetadata uploadFileMetadata = fileMetadatas.get(keyName); List<PartETag> partETags = uploadParts.get(keyName); if (partETags == null || uploadFileMetadata == null || uploadFileMetadata.getFileMetadata().getNumberOfBlocks() != partETags.size()) { return; } if (partETags.size() <= 1) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); return; } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, uploadFileMetadata.getUploadId(), partETags); CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest); if (result.getETag() != null) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); } }
@Override public void close() throws IOException { open = false; if (uploadBuffer.remaining() > 0) { flush(); } CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest() .withBucketName(path.getBucket()) .withKey(path.getKey()) .withUploadId(uploadId) .withPartETags(eTags); try { amazonS3.completeMultipartUpload(request); } catch (AmazonClientException e) { throw new IOException(e); } }
@Override public CompleteMultipartUploadResult completeMultipartUploadSecurely( CompleteMultipartUploadRequest req) { appendUserAgent(req, USER_AGENT); String uploadId = req.getUploadId(); final T uploadContext = multipartUploadContexts.get(uploadId); if (uploadContext != null && !uploadContext.hasFinalPartBeenSeen()) { throw new SdkClientException( "Unable to complete an encrypted multipart upload without being told which part was the last. " + "Without knowing which part was the last, the encrypted data in Amazon S3 is incomplete and corrupt."); } CompleteMultipartUploadResult result = s3.completeMultipartUpload(req); // In InstructionFile mode, we want to write the instruction file only // after the whole upload has completed correctly. if (uploadContext != null && cryptoConfig.getStorageMode() == InstructionFile) { // Put the instruction file into S3 s3.putObject(createInstructionPutRequest( uploadContext.getBucketName(), uploadContext.getKey(), uploadContext.getContentCryptoMaterial())); } multipartUploadContexts.remove(uploadId); return result; }
partETags.add(s3.uploadPart(uploadPartRequest).getPartETag()); .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), multipartUploadId, partETags)); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult;
InitiateMultipartUploadResult initResponse = this.s3Service.initiateMultipartUpload(initRequest); partETags.add(this.s3Service.uploadPart(uploadRequest).getPartETag()); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(this.name, req.getKey(), initResponse.getUploadId(), partETags); s3Service.completeMultipartUpload(compRequest); } catch (Exception e) { SDFSLogger.getLog().warn("unable to upload object " + req.getKey(), e);
/** * Notified from * {@link AmazonS3EncryptionClient#uploadObject(UploadObjectRequest)} when * all parts have been successfully uploaded to S3. This method is * responsible for finishing off the upload by making a complete multi-part * upload request to S3 with the given list of etags. * * @param partETags * all the etags returned from S3 for the previous part uploads. * * @return the completed multi-part upload result */ public CompleteMultipartUploadResult onCompletion(List<PartETag> partETags) { return s3.completeMultipartUpload( new CompleteMultipartUploadRequest( req.getBucketName(), req.getKey(), uploadId, partETags)); }
@Override public CopyResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getDestinationBucketName(), origReq.getDestinationKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { monitor.reportFailure(); publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } CopyResult copyResult = new CopyResult(); copyResult.setSourceBucketName(origReq.getSourceBucketName()); copyResult.setSourceKey(origReq.getSourceKey()); copyResult.setDestinationBucketName(res .getBucketName()); copyResult.setDestinationKey(res.getKey()); copyResult.setETag(res.getETag()); copyResult.setVersionId(res.getVersionId()); monitor.copyComplete(); return copyResult; }
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
public CompleteMultipartUploadRequest getCompleteMultipartUploadRequest() { return new CompleteMultipartUploadRequest(this.bucketName, this.key, this.uploadId, this.partETags); }