private static void downloadFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); final String localFile = params.getRequired("localFile"); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); try { tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) { final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile); fileDownload.waitForCompletion(); RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription()); return null; } else { final Download download = mgr.download(this.bucket, this.path, localFile); download.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription()); } }); download.waitForCompletion(); return null; } }
if (downloadCount % MAX_KEYS_PER_S3_OP == 0) { for (Map.Entry<File, Download> download : downloads.entrySet()) { download.getValue().waitForCompletion(); download.getKey().setLastModified( download.getValue().getObjectMetadata().getLastModified().getTime() ); download.getValue().waitForCompletion(); download.getKey().setLastModified( download.getValue().getObjectMetadata().getLastModified().getTime() );
Download appDownload = tx.download(bucketName, key, file); try { LOGGER.info("Transfer: " + appDownload.getDescription()); LOGGER.info(" State: " + appDownload.getState()); LOGGER.info(" Progress: "); while (!appDownload.isDone()) { LOGGER.info(" transferred: " + (int) (appDownload.getProgress().getPercentTransferred() + 0.5) + "%"); CommonUtils.pause(pollingInterval); LOGGER.info(" State: " + appDownload.getState());
protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3, DataTO destData, NfsTO destImageStore) { final String storagePath = destImageStore.getUrl(); final String destPath = destData.getPath(); try { String downloadPath = determineStorageTemplatePath(storagePath, destPath, _nfsVersion); final File downloadDirectory = _storage.getFile(downloadPath); if (downloadDirectory.exists()) { s_logger.debug("Directory " + downloadPath + " already exists"); } else { if (!downloadDirectory.mkdirs()) { final String errMsg = "Unable to create directory " + downloadPath + " to copy from S3 to cache."; s_logger.error(errMsg); return new CopyCmdAnswer(errMsg); } } File destFile = new File(downloadDirectory, substringAfterLast(srcData.getPath(), S3Utils.SEPARATOR)); S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), destFile).waitForCompletion(); return postProcessing(destFile, downloadPath, destPath, srcData, destData); } catch (Exception e) { final String errMsg = format("Failed to download" + "due to $1%s", e.getMessage()); s_logger.error(errMsg, e); return new CopyCmdAnswer(errMsg); } }
File outputFile = new File("/path/to/file"); Download download = transferManager.download(req, outputFile); download.waitForCompletion(); return new FileInputStream(outputFile);
private void multiPartDownload(String keyName, File f) throws AmazonServiceException, AmazonClientException, InterruptedException { try { Download myDownload = tx.download(this.name, keyName, f); myDownload.waitForCompletion(); } finally { } }
private void downloadFile(TransferManager tx, String bucketName, String sourcePrefixKey, String destinationFile) throws Exception{ try { final File snapshotFile = new File(destinationFile); // Only create parent directory once, if it doesn't exist. final File parentDir = new File(snapshotFile.getParent()); if (!parentDir.isDirectory()) { final boolean parentDirCreated = parentDir.mkdirs(); if (!parentDirCreated) { LOGGER.error( "Error creating parent directory for file: {}. Skipping to next", destinationFile); return; } } snapshotFile.createNewFile(); final Download download = tx.download(bucketName, sourcePrefixKey, snapshotFile); download.waitForCompletion(); } catch (Exception e) { LOGGER.error("Error downloading the file {} : {}", destinationFile, e); throw new Exception(e); } }
Download download = tm.download(bucket, prefix, configFile); try { download.waitForCompletion(); } catch (InterruptedException e) { throw new IOException(e);