void checkSuccessfulUpload( CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { logger.info( "Uploaded file: {}, object eTag: {}", localPath, resultS3MultiPartUploadComplete.getETag()); } else { throw new BackupRestoreException( "Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + localPath); } }
CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() ); em.update( entity );
uploadResult.setBucketName(res.getBucketName()); uploadResult.setKey(res.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); return uploadResult;
s3.completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[]{ffFilename, completeResult.getETag(), currentState.getUploadId()}); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); if (completeResult.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, completeResult.getETag());
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { monitor.uploadFailure(); publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
@Override public CopyResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getDestinationBucketName(), origReq.getDestinationKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { monitor.reportFailure(); publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } CopyResult copyResult = new CopyResult(); copyResult.setSourceBucketName(origReq.getSourceBucketName()); copyResult.setSourceKey(origReq.getSourceKey()); copyResult.setDestinationBucketName(res .getBucketName()); copyResult.setDestinationKey(res.getKey()); copyResult.setETag(res.getETag()); copyResult.setVersionId(res.getVersionId()); monitor.copyComplete(); return copyResult; }
uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult;
/** * Completes the multipart upload and returns the result. */ private UploadResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), uploadId, collectPartETags())); uploadComplete(); UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult; }
/** * Completes the multipart upload and returns the result. */ private CopyResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest( copyObjectRequest.getDestinationBucketName(), copyObjectRequest.getDestinationKey(), uploadId, collectPartETags())); copyComplete(); CopyResult copyResult = new CopyResult(); copyResult.setSourceBucketName(copyObjectRequest.getSourceBucketName()); copyResult.setSourceKey(copyObjectRequest.getSourceKey()); copyResult.setDestinationBucketName(completeMultipartUploadResult .getBucketName()); copyResult.setDestinationKey(completeMultipartUploadResult.getKey()); copyResult.setETag(completeMultipartUploadResult.getETag()); copyResult.setVersionId(completeMultipartUploadResult.getVersionId()); return copyResult; }
@Override public PathHandle complete(Path filePath, List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId) throws IOException { byte[] uploadIdBytes = uploadId.toByteArray(); checkUploadId(uploadIdBytes); if (handles.isEmpty()) { throw new IOException("Empty upload"); } final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper(); String key = s3a.pathToKey(filePath); String uploadIdStr = new String(uploadIdBytes, 0, uploadIdBytes.length, Charsets.UTF_8); ArrayList<PartETag> eTags = new ArrayList<>(); eTags.ensureCapacity(handles.size()); long totalLength = 0; for (Pair<Integer, PartHandle> handle : handles) { byte[] payload = handle.getRight().toByteArray(); Pair<Long, String> result = parsePartHandlePayload(payload); totalLength += result.getLeft(); eTags.add(new PartETag(handle.getLeft(), result.getRight())); } AtomicInteger errorCount = new AtomicInteger(0); CompleteMultipartUploadResult result = writeHelper.completeMPUwithRetries( key, uploadIdStr, eTags, totalLength, errorCount); byte[] eTag = result.getETag().getBytes(Charsets.UTF_8); return (PathHandle) () -> ByteBuffer.wrap(eTag); }
/** * Send the CompleteMultipartUploadRequest to S3 if all the blocks of a file are uploaded into S3. * @param keyName file to upload into S3 */ private void verifyAndEmitFileMerge(String keyName) { if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) { return; } S3InitiateFileUploadOperator.UploadFileMetadata uploadFileMetadata = fileMetadatas.get(keyName); List<PartETag> partETags = uploadParts.get(keyName); if (partETags == null || uploadFileMetadata == null || uploadFileMetadata.getFileMetadata().getNumberOfBlocks() != partETags.size()) { return; } if (partETags.size() <= 1) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); return; } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, uploadFileMetadata.getUploadId(), partETags); CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest); if (result.getETag() != null) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); } }
uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult;
uploadResult.setBucketName(res.getBucketName()); uploadResult.setKey(res.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); return uploadResult;
s3.completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[]{ffFilename, completeResult.getETag(), currentState.getUploadId()}); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); if (completeResult.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, completeResult.getETag());
/** * Completes the multipart upload and returns the result. */ private UploadResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), uploadId, collectPartETags())); uploadComplete(); UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult; }
message.setHeader(S3Constants.E_TAG, uploadResult.getETag()); if (uploadResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
/** * Completes the multipart upload and returns the result. */ private CopyResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest( copyObjectRequest.getDestinationBucketName(), copyObjectRequest.getDestinationKey(), uploadId, collectPartETags())); copyComplete(); CopyResult copyResult = new CopyResult(); copyResult.setSourceBucketName(copyObjectRequest.getSourceBucketName()); copyResult.setSourceKey(copyObjectRequest.getSourceKey()); copyResult.setDestinationBucketName(completeMultipartUploadResult .getBucketName()); copyResult.setDestinationKey(completeMultipartUploadResult.getKey()); copyResult.setETag(completeMultipartUploadResult.getETag()); copyResult.setVersionId(completeMultipartUploadResult.getVersionId()); return copyResult; }
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
@Override public CopyResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getDestinationBucketName(), origReq.getDestinationKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } CopyResult copyResult = new CopyResult(); copyResult.setSourceBucketName(origReq.getSourceBucketName()); copyResult.setSourceKey(origReq.getSourceKey()); copyResult.setDestinationBucketName(res .getBucketName()); copyResult.setDestinationKey(res.getKey()); copyResult.setETag(res.getETag()); copyResult.setVersionId(res.getVersionId()); monitor.copyComplete(); return copyResult; }