Refine search
partETags.add(s3.uploadPart(uploadPartRequest).getPartETag()); new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), multipartUploadId, partETags) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) CompleteMultipartUploadResult res = s3.completeMultipartUpload(req); uploadResult.setBucketName(res.getBucketName()); uploadResult.setKey(res.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); return uploadResult;
/** * @see com.amazonaws.services.s3.model.CompleteMultipartUploadResult#getExpirationTime() */ @Override public Date getExpirationTime() { return result == null ? null : result.getExpirationTime(); }
@Override protected void doStartElement( String uri, String name, String qName, Attributes attrs) { if (atTopLevel()) { if (name.equals("CompleteMultipartUploadResult")) { result = new CompleteMultipartUploadResult(); } } }
result.setLocation(getText()); } else if (name.equals("Bucket")) { result.setBucketName(getText()); } else if (name.equals("Key")) { result.setKey(getText()); } else if (name.equals("ETag")) { result.setETag(ServiceUtils.removeQuotes(getText()));
om.setContentType( mimeType ); PutObjectResult result = null; result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om ); InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest ); .withPartSize(partSize) .withLastPart(isLastPart); partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() ); partNumber++; new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(), partETags ); CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() ); em.update( entity );
/** * Completes the multipart upload and returns the result. */ private UploadResult completeMultipartUpload() { CompleteMultipartUploadResult completeMultipartUploadResult = s3 .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), uploadId, collectPartETags())); uploadComplete(); UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult; }
final InitiateMultipartUploadResult initResponse = getEndpoint().getS3Client().initiateMultipartUpload(initRequest); final long contentLength = objectMetadata.getContentLength(); final List<PartETag> partETags = new ArrayList<PartETag>(); partETags.add(getEndpoint().getS3Client().uploadPart(uploadRequest).getPartETag()); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(getConfiguration().getBucketName(), keyName, initResponse.getUploadId(), partETags); uploadResult = getEndpoint().getS3Client().completeMultipartUpload(compRequest); message.setHeader(S3Constants.E_TAG, uploadResult.getETag()); if (uploadResult.getVersionId() != null) { message.setHeader(S3Constants.VERSION_ID, uploadResult.getVersionId());
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { monitor.uploadFailure(); publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( bucket, key, currentState.getUploadId(), currentState.getPartETags()); try { s3.completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[]{ffFilename, completeResult.getETag(), currentState.getUploadId()}); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); if (completeResult.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, completeResult.getETag()); if (completeResult.getExpirationTime() != null) { attributes.put(S3_EXPIRATION_ATTR_KEY, completeResult.getExpirationTime().toString());
/** * Send the CompleteMultipartUploadRequest to S3 if all the blocks of a file are uploaded into S3. * @param keyName file to upload into S3 */ private void verifyAndEmitFileMerge(String keyName) { if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) { return; } S3InitiateFileUploadOperator.UploadFileMetadata uploadFileMetadata = fileMetadatas.get(keyName); List<PartETag> partETags = uploadParts.get(keyName); if (partETags == null || uploadFileMetadata == null || uploadFileMetadata.getFileMetadata().getNumberOfBlocks() != partETags.size()) { return; } if (partETags.size() <= 1) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); return; } CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, uploadFileMetadata.getUploadId(), partETags); CompleteMultipartUploadResult result = s3Client.completeMultipartUpload(compRequest); if (result.getETag() != null) { uploadedFiles.add(keyName); LOG.debug("Uploaded file {} successfully", keyName); } }
void checkSuccessfulUpload( CompleteMultipartUploadResult resultS3MultiPartUploadComplete, Path localPath) throws BackupRestoreException { if (null != resultS3MultiPartUploadComplete && null != resultS3MultiPartUploadComplete.getETag()) { logger.info( "Uploaded file: {}, object eTag: {}", localPath, resultS3MultiPartUploadComplete.getETag()); } else { throw new BackupRestoreException( "Error uploading file as ETag or CompleteMultipartUploadResult is NULL -" + localPath); } }
@Override public String getVersionId() { return result == null ? null : result.getVersionId(); }
/** * @see com.amazonaws.services.s3.model.CompleteMultipartUploadResult#getExpirationTimeRuleId() */ @Override public String getExpirationTimeRuleId() { return result == null ? null : result.getExpirationTimeRuleId(); }
/** * @see com.amazonaws.services.s3.model.CompleteMultipartUploadResult#isRequesterCharged() */ public boolean isRequesterCharged() { return result == null ? false : result.isRequesterCharged(); }
partETags.add(s3.uploadPart(uploadPartRequest).getPartETag()); .completeMultipartUpload(new CompleteMultipartUploadRequest(putObjectRequest .getBucketName(), putObjectRequest.getKey(), multipartUploadId, partETags)); uploadResult.setBucketName(completeMultipartUploadResult.getBucketName()); uploadResult.setKey(completeMultipartUploadResult.getKey()); uploadResult.setETag(completeMultipartUploadResult.getETag()); uploadResult.setVersionId(completeMultipartUploadResult.getVersionId()); return uploadResult;
@Override public UploadResult call() throws Exception { CompleteMultipartUploadResult res; try { CompleteMultipartUploadRequest req = new CompleteMultipartUploadRequest( origReq.getBucketName(), origReq.getKey(), uploadId, collectPartETags()) .withRequesterPays(origReq.isRequesterPays()) .withGeneralProgressListener(origReq.getGeneralProgressListener()) .withRequestMetricCollector(origReq.getRequestMetricCollector()) ; res = s3.completeMultipartUpload(req); } catch (Exception e) { publishProgress(listener, ProgressEventType.TRANSFER_FAILED_EVENT); throw e; } UploadResult uploadResult = new UploadResult(); uploadResult.setBucketName(origReq .getBucketName()); uploadResult.setKey(origReq.getKey()); uploadResult.setETag(res.getETag()); uploadResult.setVersionId(res.getVersionId()); monitor.uploadComplete(); return uploadResult; }
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( bucket, key, currentState.getUploadId(), currentState.getPartETags()); try { s3.completeMultipartUpload(completeRequest); getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", new Object[]{ffFilename, completeResult.getETag(), currentState.getUploadId()}); if (completeResult.getVersionId() != null) { attributes.put(S3_VERSION_ATTR_KEY, completeResult.getVersionId()); if (completeResult.getETag() != null) { attributes.put(S3_ETAG_ATTR_KEY, completeResult.getETag()); if (completeResult.getExpirationTime() != null) { attributes.put(S3_EXPIRATION_ATTR_KEY, completeResult.getExpirationTime().toString());
result.setLocation(getText()); } else if (name.equals("Bucket")) { result.setBucketName(getText()); } else if (name.equals("Key")) { result.setKey(getText()); } else if (name.equals("ETag")) { result.setETag(ServiceUtils.removeQuotes(getText()));
@Override public PathHandle complete(Path filePath, List<Pair<Integer, PartHandle>> handles, UploadHandle uploadId) throws IOException { byte[] uploadIdBytes = uploadId.toByteArray(); checkUploadId(uploadIdBytes); if (handles.isEmpty()) { throw new IOException("Empty upload"); } final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper(); String key = s3a.pathToKey(filePath); String uploadIdStr = new String(uploadIdBytes, 0, uploadIdBytes.length, Charsets.UTF_8); ArrayList<PartETag> eTags = new ArrayList<>(); eTags.ensureCapacity(handles.size()); long totalLength = 0; for (Pair<Integer, PartHandle> handle : handles) { byte[] payload = handle.getRight().toByteArray(); Pair<Long, String> result = parsePartHandlePayload(payload); totalLength += result.getLeft(); eTags.add(new PartETag(handle.getLeft(), result.getRight())); } AtomicInteger errorCount = new AtomicInteger(0); CompleteMultipartUploadResult result = writeHelper.completeMPUwithRetries( key, uploadIdStr, eTags, totalLength, errorCount); byte[] eTag = result.getETag().getBytes(Charsets.UTF_8); return (PathHandle) () -> ByteBuffer.wrap(eTag); }
@Override public String getVersionId() { return result == null ? null : result.getVersionId(); }