ObjectMetadata meta = new ObjectMetadata(); if (mSseEnabled) { meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); meta.setContentMD5(new String(Base64.encode(mHash.digest()))); meta.setContentLength(mFile.length()); PutObjectRequest putReq = new PutObjectRequest(mBucketName, path, mFile).withMetadata(meta); mManager.upload(putReq).waitForUploadResult(); if (!mFile.delete()) { LOG.error("Failed to delete temporary file @ {}", mFile.getPath());
public static void uploadDirWithSubprogress(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); TransferManager xfer_mgr = new TransferManager(); try { MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory( bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showMultiUploadProgress(multi_upload); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(multi_upload); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
@Override protected boolean copyObject(String src, String dst) { LOG.debug("Copying {} to {}", src, dst); // Retry copy for a few times, in case some AWS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { CopyObjectRequest request = new CopyObjectRequest(mBucketName, src, mBucketName, dst); if (Boolean.parseBoolean( mConf.get(PropertyKey.UNDERFS_S3A_SERVER_SIDE_ENCRYPTION_ENABLED))) { ObjectMetadata meta = new ObjectMetadata(); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(meta); } mManager.copy(request).waitForCopyResult(); return true; } catch (AmazonClientException | InterruptedException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
private static void downloadFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); final String localFile = params.getRequired("localFile"); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); try { tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
@Override public void close() throws IOException { if (closed.getAndSet(true)) { return; } mBackupOutputStream.close(); LOG.debug("OutputStream for key '{}' closed. Now beginning upload", mKey); try { final ObjectMetadata om = new ObjectMetadata(); om.setContentLength(mBackupFile.length()); om.setContentType(mContentType); om.setUserMetadata(mMetadata); PutObjectRequest putObjectRequest = new PutObjectRequest(mBucketName, mKey, mBackupFile); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); upload.waitForUploadResult(); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException(e.toString()) .initCause(e); } catch (AmazonClientException e) { throw new IOException(String.format("saving output %s %s", mKey, e)); } finally { if (!mBackupFile.delete()) { LOG.warn("Could not delete temporary cos file: {}", mBackupOutputStream); } super.close(); } LOG.debug("OutputStream for key '{}' upload complete", mKey); }
ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentEncoding("UTF-8"); size = inputStream.available(); metadata.setContentLength(size); TransferManager transferManager = new TransferManager(credentialsProvider); Upload upload = transferManager.upload(bucket_name, key, images3, metadata); upload.waitForCompletion();
HashBlobArchive.close(); this.updateObject(binm, nmd); else s3Service.putObject(this.name, binm, new ByteArrayInputStream(sz), nmd); } catch (AmazonS3Exception e1) { if (e1.getStatusCode() == 409) { this.updateObject(binm, nmd); else s3Service.putObject(this.name, binm, new ByteArrayInputStream(sz), nmd); } catch (Exception e2) { throw new IOException(e2); } finally { try { tx.shutdownNow(false); s3Service.shutdown(); } catch (Exception e) {
STATS.uploadStarted(); PutObjectRequest request = new PutObjectRequest(host, key, tempFile); if (sseEnabled) { switch (sseType) { case KMS: if (sseKmsKeyId != null) { request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(sseKmsKeyId)); request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams()); ObjectMetadata metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setMetadata(metadata); break; request.withCannedAcl(aclType); Upload upload = transferManager.upload(request);
@Override public void put(String uri, File f) throws AmazonClientException { LOG.trace("Uploading " + uri); String[] parts = pieces(uri); ObjectMetadata om = new ObjectMetadata(); om.setContentLength(f.length()); if (f.getName().endsWith("gzip")) { om.setContentEncoding("gzip"); } uploadsInProgress.incrementAndGet(); try { PutObjectRequest req = new PutObjectRequest(parts[0],parts[1],f); req.setMetadata(om); UploadResult resp = svc.upload(req).waitForUploadResult(); LOG.trace("Uploaded " + uri + " with ETag " + resp.getETag()); } catch (InterruptedException ie) { LOG.error("Interrupted while uploading {} to {}.", f.getPath(), uri); throw Throwables.propagate(ie); } finally { uploadsInProgress.decrementAndGet(); } }
@Override public void addMetadataRecord(final InputStream input, final String name) throws DataStoreException { checkArgument(input != null, "input should not be null"); checkArgument(!Strings.isNullOrEmpty(name), "name should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); Upload upload = tmx.upload(s3ReqDecorator .decorate(new PutObjectRequest(bucket, addMetaKeyPrefix(name), input, new ObjectMetadata()))); upload.waitForUploadResult(); } catch (InterruptedException e) { LOG.error("Error in uploading", e); throw new DataStoreException("Error in uploading", e); } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
transferConfiguration.setMultipartUploadThreshold(partSizeThreshold); TransferManager transfers = new TransferManager(client); transfers.setConfiguration(transferConfiguration); final ObjectMetadata om = new ObjectMetadata(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { om.setServerSideEncryption(serverSideEncryptionAlgorithm); PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile); putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred(); if (statistics != null && delta != 0) { if (LOG.isDebugEnabled()) {
ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(contentLength); .runInterruptible(() -> { try (InputStream in = payload.open()) { PutObjectRequest req = new PutObjectRequest(bucket, key, in, meta); UploadResult result = transferManager.upload(req).waitForUploadResult();
String suffix = StringUtils.substringAfter(fullPath, getPath(localUri)); String key = getPath(s3Uri) + suffix; ObjectMetadata metadata = new ObjectMetadata(); metadata.setLastModified(new Date(fileToCopy.lastModified())); PutObjectRequest request = new PutObjectRequest(bucket, key, fileToCopy).withMetadata(metadata); uploads.add(tx.upload(request)); uploadCount++; metrics.s3FilesUploaded.mark(1); uploads.clear(); tx.shutdownNow(false); if (deleteCount % MAX_KEYS_PER_S3_OP == 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); logger.info(String.format("Deleted %s files", deleteCount)); keys.clear(); s3.deleteObjects(deleteObjectsRequest); logger.info(String.format("Deleted %s files", deleteCount));
private void addFile() throws Exception { InputStream is = new ByteArrayInputStream(this.outputStream.toByteArray()); int contentLength = outputStream.size(); TransferManager transferManager = new TransferManager(amazonS3Client); ObjectMetadata metadata = new ObjectMetadata(); metadata.setExpirationTime(DateTime.now().plusDays(365 * 3).toDate()); metadata.setContentLength(contentLength); metadata.addUserMetadata("writer", "org.apache.streams"); for (String s : metaData.keySet()) { metadata.addUserMetadata(s, metaData.get(s)); } String fileNameToWrite = path + fileName; Upload upload = transferManager.upload(bucketName, fileNameToWrite, is, metadata); try { upload.waitForUploadResult(); is.close(); transferManager.shutdownNow(false); LOGGER.info("S3 File Close[{} kb] - {}", contentLength / 1024, path + fileName); } catch (Exception ignored) { LOGGER.trace("Ignoring", ignored); } }
final ObjectMetadata om = new ObjectMetadata(); if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) { om.setServerSideEncryption(serverSideEncryptionAlgorithm); PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile); putObjectRequest.setCannedAcl(cannedACL); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred(); if (statistics != null && delta != 0) { if (LOG.isDebugEnabled()) {
private void uploadObject() throws IOException { try { log.debug("Starting upload for host: %s, key: %s, file: %s, size: %s", host, key, tempFile, tempFile.length()); STATS.uploadStarted(); PutObjectRequest request = new PutObjectRequest(host, key, tempFile); if (sseEnabled) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setMetadata(metadata); } Upload upload = transferManager.upload(request); if (log.isDebugEnabled()) { upload.addProgressListener(createProgressListener(upload)); } upload.waitForCompletion(); STATS.uploadSuccessful(); log.debug("Completed upload for host: %s, key: %s", host, key); } catch (AmazonClientException e) { STATS.uploadFailed(); throw new IOException(e); } catch (InterruptedException e) { STATS.uploadFailed(); Thread.currentThread().interrupt(); throw new InterruptedIOException(); } }
return upload(new PutObjectRequest(bucketName, key, file));
private boolean upload(AmazonS3 s3, File sourceFile) throws MojoExecutionException TransferManager mgr = new TransferManager(s3); transfer = mgr.upload(new PutObjectRequest(bucketName, destination, sourceFile) .withCannedAcl(CannedAccessControlList.BucketOwnerFullControl)); } else if (sourceFile.isDirectory()) { transfer = mgr.uploadDirectory(bucketName, destination, sourceFile, recursive, new ObjectMetadataProvider() { @Override getLog().debug("Transferring " + transfer.getProgress().getTotalBytesToTransfer() + " bytes..."); transfer.waitForCompletion(); getLog().info("Transferred " + transfer.getProgress().getBytesTransferred() + " bytes."); } catch (InterruptedException e) { return false;
getLogger().info("Uploading... s3://{}/{}", bucketName, key); Upload upload = s3mgr.upload(new PutObjectRequest(getBucketName(), getKey(), getFile()) .withMetadata(getObjectMetadata())); upload.addProgressListener(new ProgressListener() { setResourceUrl(s3.getUrl(bucketName, key).toString()); getLogger().info("Upload completed: {}", getResourceUrl());