public PrestoS3OutputStream( AmazonS3 s3, String host, String key, File tempFile, boolean sseEnabled, PrestoS3SseType sseType, String sseKmsKeyId, long multiPartUploadMinFileSize, long multiPartUploadMinPartSize, PrestoS3AclType aclType) throws IOException { super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null")))); transferManager = TransferManagerBuilder.standard() .withS3Client(requireNonNull(s3, "s3 is null")) .withMinimumUploadPartSize(multiPartUploadMinPartSize) .withMultipartUploadThreshold(multiPartUploadMinFileSize).build(); requireNonNull(aclType, "aclType is null"); this.aclType = aclType.getCannedACL(); this.host = requireNonNull(host, "host is null"); this.key = requireNonNull(key, "key is null"); this.tempFile = tempFile; this.sseEnabled = sseEnabled; this.sseType = requireNonNull(sseType, "sseType is null"); this.sseKmsKeyId = sseKmsKeyId; log.debug("OutputStream for key '%s' using file: %s", key, tempFile); }
.withS3Client(amazonS3Client).withExecutorFactory(() -> service) .withMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD) .build();
public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression, boolean produceReply) { this(TransferManagerBuilder.standard() .withS3Client(amazonS3) .build(), bucketExpression, produceReply); Assert.notNull(amazonS3, "'amazonS3' must not be null"); }
public S3MessageHandler(AmazonS3 amazonS3, String bucket, boolean produceReply) { this(TransferManagerBuilder.standard() .withS3Client(amazonS3) .build(), bucket, produceReply); Assert.notNull(amazonS3, "'amazonS3' must not be null"); }
public S3MessageHandler(AmazonS3 amazonS3, Expression bucketExpression, boolean produceReply) { this(TransferManagerBuilder.standard() .withS3Client(amazonS3) .build(), bucketExpression, produceReply); Assert.notNull(amazonS3, "'amazonS3' must not be null"); }
public LogsS3Uploader(AmazonS3 s3Client, String s3Bucket) { this.s3Bucket = s3Bucket; this.transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); }
public PrestoS3OutputStream( AmazonS3 s3, String host, String key, File tempFile, boolean sseEnabled, PrestoS3SseType sseType, String sseKmsKeyId, long multiPartUploadMinFileSize, long multiPartUploadMinPartSize, PrestoS3AclType aclType) throws IOException { super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null")))); transferManager = TransferManagerBuilder.standard() .withS3Client(requireNonNull(s3, "s3 is null")) .withMinimumUploadPartSize(multiPartUploadMinPartSize) .withMultipartUploadThreshold(multiPartUploadMinFileSize).build(); requireNonNull(aclType, "aclType is null"); this.aclType = aclType.getCannedACL(); this.host = requireNonNull(host, "host is null"); this.key = requireNonNull(key, "key is null"); this.tempFile = tempFile; this.sseEnabled = sseEnabled; this.sseType = requireNonNull(sseType, "sseType is null"); this.sseKmsKeyId = sseKmsKeyId; log.debug("OutputStream for key '%s' using file: %s", key, tempFile); }
private TransferManager createTransferManager() { return TransferManagerBuilder.standard().withS3Client(getS3Client()).build(); } }
List<S3ObjectSummary> summaries = listKeys(s3Uri); String bucket = s3Uri.getHost(); TransferManager tx = TransferManagerBuilder.standard().withS3Client(s3).build(); Map<File, Download> downloads = new HashMap<>(); logger.info(String.format("Downloading files from %s to %s", s3Uri.toString(), localUri.toString()));
TransferManager xferMgr = TransferManagerBuilder.standard().withS3Client(s3client).build(); try { MultipleFileUpload xfer = xferMgr.uploadDirectory(s3Bucket,
public S3FilePersistence (String region) { amazonS3 = AmazonS3ClientBuilder.standard() // .enableAccelerateMode() // this fails looking up s3-accelerate.amazonaws.com .withRegion(region) .build(); transferManager = TransferManagerBuilder.standard() .withS3Client(amazonS3) .build(); }
public S3FilePersistence (String region) { amazonS3 = AmazonS3ClientBuilder.standard() // .enableAccelerateMode() // this fails looking up s3-accelerate.amazonaws.com .withRegion(region) .build(); transferManager = TransferManagerBuilder.standard() .withS3Client(amazonS3) .build(); }
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) { final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile); fileDownload.waitForCompletion(); RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription()); return null; } else { final Download download = mgr.download(this.bucket, this.path, localFile); download.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription()); } }); download.waitForCompletion(); return null; } }
public TransferManager newInstance(AmazonS3 targetS3Client, S3S3CopierOptions s3s3CopierOptions) { return TransferManagerBuilder .standard() .withMultipartCopyThreshold(s3s3CopierOptions.getMultipartCopyThreshold()) .withMultipartCopyPartSize(s3s3CopierOptions.getMultipartCopyPartSize()) .withS3Client(targetS3Client) .build(); }
public TransferManager newInstance(AmazonS3 targetS3Client, S3S3CopierOptions s3s3CopierOptions) { return TransferManagerBuilder .standard() .withMultipartCopyThreshold(s3s3CopierOptions.getMultipartCopyThreshold()) .withMultipartCopyPartSize(s3s3CopierOptions.getMultipartCopyPartSize()) .withS3Client(targetS3Client) .build(); }
.withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars)) .build(); final Copy copy = mgr.copy(request);
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); final MultipleFileUpload fileUpload;
AmazonS3 s3 = ext.getClient(); TransferManager s3mgr = TransferManagerBuilder.standard().withS3Client(s3).build(); getLogger().info("Uploading... s3://{}/{}", bucketName, key);
/** * Implementation of the Mapper::setup() method. This extracts the S3MapReduceCp options specified in the Job's * configuration, to set up the Job. * * @param context Mapper's context. * @throws IOException On IO failure. * @throws InterruptedException If the job is interrupted. */ @Override public void setup(Context context) throws IOException, InterruptedException { conf = new S3MapReduceCpConfiguration(context.getConfiguration()); ignoreFailures = conf.getBoolean(ConfigurationVariable.IGNORE_FAILURES); targetFinalPath = new Path(conf.get(S3MapReduceCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); AwsS3ClientFactory awsS3ClientFactory = new AwsS3ClientFactory(); transferManager = TransferManagerBuilder .standard() .withMinimumUploadPartSize(conf.getLong(ConfigurationVariable.MINIMUM_UPLOAD_PART_SIZE)) .withMultipartUploadThreshold(conf.getLong(ConfigurationVariable.MULTIPART_UPLOAD_THRESHOLD)) .withS3Client(awsS3ClientFactory.newInstance(conf)) .withShutDownThreadPools(true) .withExecutorFactory(new ExecutorFactory() { @Override public ExecutorService newExecutor() { return Executors.newFixedThreadPool(conf.getInt(ConfigurationVariable.NUMBER_OF_UPLOAD_WORKERS)); } }) .build(); }
/** * Implementation of the Mapper::setup() method. This extracts the S3MapReduceCp options specified in the Job's * configuration, to set up the Job. * * @param context Mapper's context. * @throws IOException On IO failure. * @throws InterruptedException If the job is interrupted. */ @Override public void setup(Context context) throws IOException, InterruptedException { conf = new S3MapReduceCpConfiguration(context.getConfiguration()); ignoreFailures = conf.getBoolean(ConfigurationVariable.IGNORE_FAILURES); targetFinalPath = new Path(conf.get(S3MapReduceCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); AwsS3ClientFactory awsS3ClientFactory = new AwsS3ClientFactory(); transferManager = TransferManagerBuilder .standard() .withMinimumUploadPartSize(conf.getLong(ConfigurationVariable.MINIMUM_UPLOAD_PART_SIZE)) .withMultipartUploadThreshold(conf.getLong(ConfigurationVariable.MULTIPART_UPLOAD_THRESHOLD)) .withS3Client(awsS3ClientFactory.newInstance(conf)) .withShutDownThreadPools(true) .withExecutorFactory(new ExecutorFactory() { @Override public ExecutorService newExecutor() { return Executors.newFixedThreadPool(conf.getInt(ConfigurationVariable.NUMBER_OF_UPLOAD_WORKERS)); } }) .build(); }