Refine search
/** * Sets the list of part numbers and ETags that identify the individual * parts of the multipart upload to complete based on the specified results * from part uploads. * * @param uploadPartResults * The list of results from the individual part uploads in the * multipart upload to complete. * * @return This updated CompleteMultipartUploadRequest object. */ public CompleteMultipartUploadRequest withPartETags(UploadPartResult... uploadPartResults) { for (UploadPartResult result : uploadPartResults) { this.partETags.add(new PartETag(result.getPartNumber(), result.getETag())); } return this; }
@Override public String toString() { StringBuilder buf = new StringBuilder(128); buf.append("S3Recoverable: "); buf.append("key=").append(objectName); buf.append(", uploadId=").append(uploadId); buf.append(", bytesInParts=").append(numBytesInParts); buf.append(", parts=["); int num = 0; for (PartETag part : parts) { if (0 != num++) { buf.append(", "); } buf.append(part.getPartNumber()).append('=').append(part.getETag()); } buf.append("], trailingPart=").append(lastPartObject); buf.append("trailingPartLen=").append(lastPartObjectLength); return buf.toString(); } }
private PartETag uploadChunk(byte[] buffer, int bytesRead, InitiateMultipartUploadResult initResponse, int index) throws Exception { byte[] md5 = S3Utils.md5(buffer, bytesRead); UploadPartRequest request = new UploadPartRequest(); request.setBucketName(initResponse.getBucketName()); request.setKey(initResponse.getKey()); request.setUploadId(initResponse.getUploadId()); request.setPartNumber(index); request.setPartSize(bytesRead); request.setMd5Digest(S3Utils.toBase64(md5)); request.setInputStream(new ByteArrayInputStream(buffer, 0, bytesRead)); UploadPartResult response = s3Client.uploadPart(request); PartETag partETag = response.getPartETag(); if ( !response.getPartETag().getETag().equals(S3Utils.toHex(md5)) ) { throw new Exception("Unable to match MD5 for part " + index); } return partETag; }
currentState.getContentLength(), currentState.getPartETags().size(), Integer.toString(lastETag.getPartNumber()), lastETag.getETag()}); } else { getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + try { UploadPartResult uploadPartResult = s3.uploadPart(uploadRequest); currentState.addPartETag(uploadPartResult.getPartETag()); currentState.setFilePosition(currentState.getFilePosition() + thisPartSize); try { uploadPartResult.getETag(), currentState.getUploadId()}); } catch (AmazonClientException e) { getLogger().info("Failure uploading part flowfile={} part={} bucket={} key={} " +
/** * Returns an identifier which identifies the copy part by its part number * and the entity tag computed from the part's data. This information is * later needed to complete a multipart copy. * * @return An identifier which identifies the copy part by its part number * and the entity tag computed from the part's data. */ public PartETag getPartETag() { return new PartETag(partNumber, etag); }
omd.setContentLength(partSize); PutObjectResult result = s3Client.putObject(new PutObjectRequest(bucketName, metaData.getKeyName(), bis, omd)); partETag = new PartETag(1, result.getETag()); } else { partETag = s3Client.uploadPart(uploadRequest).getPartETag(); } catch (Exception e) { throw new RuntimeException(e);
public int compare(PartETag tag1, PartETag tag2) { if (tag1.getPartNumber() < tag2.getPartNumber()) return -1; if (tag1.getPartNumber() > tag2.getPartNumber()) return 1; return 0; } });
private Void uploadPart() throws AmazonClientException, BackupRestoreException { UploadPartRequest req = new UploadPartRequest(); req.setBucketName(dataPart.getBucketName()); req.setKey(dataPart.getS3key()); req.setUploadId(dataPart.getUploadID()); req.setPartNumber(dataPart.getPartNo()); req.setPartSize(dataPart.getPartData().length); req.setMd5Digest(SystemUtils.toBase64(dataPart.getMd5())); req.setInputStream(new ByteArrayInputStream(dataPart.getPartData())); UploadPartResult res = client.uploadPart(req); PartETag partETag = res.getPartETag(); if (!partETag.getETag().equals(SystemUtils.toHex(dataPart.getMd5()))) throw new BackupRestoreException( "Unable to match MD5 for part " + dataPart.getPartNo()); partETags.add(partETag); if (this.partsUploaded != null) this.partsUploaded.incrementAndGet(); return null; }
currentState.getContentLength(), currentState.getPartETags().size(), Integer.toString(lastETag.getPartNumber()), lastETag.getETag()}); } else { getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + try { UploadPartResult uploadPartResult = s3.uploadPart(uploadRequest); currentState.addPartETag(uploadPartResult.getPartETag()); currentState.setFilePosition(currentState.getFilePosition() + thisPartSize); try { uploadPartResult.getETag(), currentState.getUploadId()}); } catch (AmazonClientException e) { getLogger().info("Failure uploading part flowfile={} part={} bucket={} key={} " +
/** * Returns an identifier which identifies the upload part by its part number * and the entity tag computed from the part's data. This information is * later needed to complete a multipart upload. * * @return An identifier which identifies the upload part by its part number * and the entity tag computed from the part's data. */ public PartETag getPartETag() { return new PartETag(partNumber, eTag); }
@Override public int compare(PartETag tag1, PartETag tag2) { if (tag1.getPartNumber() < tag2.getPartNumber()) return -1; if (tag1.getPartNumber() > tag2.getPartNumber()) return 1; return 0; } });
/** * Sets the list of part numbers and ETags that identify the individual * parts of the multipart upload to complete based on the specified results * from part uploads. * * @param uploadPartResultsCollection * The list of results from the individual part uploads in the * multipart upload to complete. * * @return This updated CompleteMultipartUploadRequest object. */ public CompleteMultipartUploadRequest withPartETags(Collection<UploadPartResult> uploadPartResultsCollection) { for (UploadPartResult result : uploadPartResultsCollection) { this.partETags.add(new PartETag(result.getPartNumber(), result.getETag())); } return this; }
@Override public PartETag call() throws Exception { // this is the queued upload operation LOG.debug("Uploading part {} for id '{}'", currentPartNumber, uploadId); // do the upload PartETag partETag; try { partETag = fs.uploadPart(request).getPartETag(); LOG.debug("Completed upload of {} to part {}", block, partETag.getETag()); } finally { // close the stream and block closeAll(LOG, uploadData, block); } return partETag; } });
public MultipartState(String buf) { String[] fields = buf.split(SEPARATOR); _uploadId = fields[0]; _filePosition = Long.parseLong(fields[1]); _partETags = new ArrayList<>(); for (String part : fields[2].split(",")) { if (part != null && !part.isEmpty()) { String[] partFields = part.split("/"); _partETags.add(new PartETag(Integer.parseInt(partFields[0]), partFields[1])); } } _partSize = Long.parseLong(fields[3]); _storageClass = StorageClass.fromValue(fields[4]); _contentLength = Long.parseLong(fields[5]); _timestamp = Long.parseLong(fields[6]); }
@Override public String toString() { StringBuilder buf = new StringBuilder(); buf.append(_uploadId).append(SEPARATOR) .append(_filePosition.toString()).append(SEPARATOR); if (_partETags.size() > 0) { boolean first = true; for (PartETag tag : _partETags) { if (!first) { buf.append(","); } else { first = false; } buf.append(String.format("%d/%s", tag.getPartNumber(), tag.getETag())); } } buf.append(SEPARATOR) .append(_partSize.toString()).append(SEPARATOR) .append(_storageClass.toString()).append(SEPARATOR) .append(_contentLength.toString()).append(SEPARATOR) .append(_timestamp.toString()); return buf.toString(); } }
public int compare(PartETag tag1, PartETag tag2) { if (tag1.getPartNumber() < tag2.getPartNumber()) return -1; if (tag1.getPartNumber() > tag2.getPartNumber()) return 1; return 0; } });
@Override public void run() { try { final UploadPartResult result = s3AccessHelper.uploadPart(objectName, uploadId, partNumber, file.getInputFile(), file.getPos()); future.complete(new PartETag(result.getPartNumber(), result.getETag())); file.release(); } catch (Throwable t) { future.completeExceptionally(t); } } }
@Override public PartETag call() throws Exception { // this is the queued upload operation LOG.debug("Uploading part {} for id '{}'", currentPartNumber, uploadId); // do the upload PartETag partETag; try { partETag = fs.uploadPart(request).getPartETag(); LOG.debug("Completed upload of {} to part {}", block, partETag.getETag()); } finally { // close the stream and block closeAll(LOG, uploadData, block); } return partETag; } });
final byte[] buffer = new byte[bb.getInt()]; bb.get(buffer); parts.add(new PartETag(partNum, new String(buffer, CHARSET)));
int partEtagBytes = 0; for (int i = 0; i < parts.length; i++) { etags[i] = parts[i].getETag().getBytes(CHARSET); partEtagBytes += etags[i].length + 2 * Integer.BYTES; for (int i = 0; i < parts.length; i++) { PartETag pe = parts[i]; bb.putInt(pe.getPartNumber()); bb.putInt(etags[i].length); bb.put(etags[i]);