/** * Forcefully shuts down this TransferManager instance - currently executing * transfers will not be allowed to finish. It also by default shuts down * the underlying Amazon S3 client. * * @see #shutdownNow(boolean) */ public void shutdownNow() { shutdownNow(true); }
@Override public void close() throws IOException { if (closed) { return; } closed = true; try { super.close(); uploadObject(); } finally { if (!tempFile.delete()) { log.warn("Could not delete temporary file: %s", tempFile); } // close transfer manager but keep underlying S3 client open transferManager.shutdownNow(false); } }
/** * Force the client cache to provide a certain client for the region which * that client is configured. This can be useful to provide clients with * different {@link com.amazonaws.services.s3.S3ClientOptions} or use a * {@link com.amazonaws.services.s3.AmazonS3EncryptionClient} in place of a * regular client. * * Using a new client will also forcibly shut down any * {@link TransferManager} that has been instantiated with that client, with * the {@link TransferManager#shutdownNow()} method. * * @param client * An {@link AmazonS3} to use in the cache. Its region will * be detected automatically. */ public void useClient(AmazonS3 client) { String region = client.getRegionName(); synchronized (transferManagersByRegion) { TransferManager tm = transferManagersByRegion.remove(region); if (tm != null) { tm.shutdownNow(); } clientsByRegion.put(region, client); } }
private static void downloadFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); final String localFile = params.getRequired("localFile"); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); try { tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
public static void copyObjectSimple(String from_bucket, String from_key, String to_bucket, String to_key) { System.out.println("Copying s3 object: " + from_key); System.out.println(" from bucket: " + from_bucket); System.out.println(" to s3 object: " + to_key); System.out.println(" in bucket: " + to_bucket); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
public static void uploadFileList(String[] file_paths, String bucket_name, String key_prefix, boolean pause) { System.out.println("file list: " + Arrays.toString(file_paths) + (pause ? " (pause)" : "")); // convert the file paths to a list of File objects (required by the // uploadFileList method) ArrayList<File> files = new ArrayList<File>(); for (String path : file_paths) { files.add(new File(path)); } TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileUpload xfer = xfer_mgr.uploadFileList(bucket_name, key_prefix, new File("."), files); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
public static void uploadDirWithSubprogress(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); TransferManager xfer_mgr = new TransferManager(); try { MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory( bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showMultiUploadProgress(multi_upload); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(multi_upload); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
public static void uploadDir(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
public static void downloadDir(String bucket_name, String key_prefix, String dir_path, boolean pause) { System.out.println("downloading to directory: " + dir_path + (pause ? " (pause)" : "")); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileDownload xfer = xfer_mgr.downloadDirectory( bucket_name, key_prefix, new File(dir_path)); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
public static void downloadFile(String bucket_name, String key_name, String file_path, boolean pause) { System.out.println("Downloading to file: " + file_path + (pause ? " (pause)" : "")); File f = new File(file_path); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Download xfer = xfer_mgr.download(bucket_name, key_name, f); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
public static void uploadFile(String file_path, String bucket_name, String key_prefix, boolean pause) { System.out.println("file: " + file_path + (pause ? " (pause)" : "")); String key_name = null; if (key_prefix != null) { key_name = key_prefix + '/' + file_path; } else { key_name = file_path; } File f = new File(file_path); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Upload xfer = xfer_mgr.upload(bucket_name, key_name, f); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); }
System.exit(1); xfer_mgr.shutdownNow();
/** * Forcefully shuts down this TransferManager instance - currently executing * transfers will not be allowed to finish. It also by default shuts down * the underlying Amazon S3 client. * * @see #shutdownNow(boolean) */ public void shutdownNow() { shutdownNow(true); }
/** * Shutdown transfer queue and release other engaged resources. */ @Override protected void cleanup(Mapper<Text, CopyListingFileStatus, Text, Text>.Context context) throws IOException, InterruptedException { if (transferManager != null) { transferManager.shutdownNow(true); } }
/** * Shutdown transfer queue and release other engaged resources. */ @Override protected void cleanup(Mapper<Text, CopyListingFileStatus, Text, Text>.Context context) throws IOException, InterruptedException { if (transferManager != null) { transferManager.shutdownNow(true); } }
@Override public void close() throws IOException { try { super.close(); } finally { if (transfers != null) { transfers.shutdownNow(true); transfers = null; } } }
@Override public void close() { // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public void close() { // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public void close() throws DataStoreException { super.close(); // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }