public static void showTransferProgress(Transfer xfer) { // print the transfer's human-readable description System.out.println(xfer.getDescription()); // print an empty progress bar... printProgressBar(0.0); // update the progress bar while the xfer is ongoing. do { try { Thread.sleep(100); } catch (InterruptedException e) { return; } // Note: so_far and total aren't used, they're just for // documentation purposes. TransferProgress progress = xfer.getProgress(); long so_far = progress.getBytesTransferred(); long total = progress.getTotalBytesToTransfer(); double pct = progress.getPercentTransferred(); eraseProgressBar(); printProgressBar(pct); } while (xfer.isDone() == false); // print the final state of the transfer. TransferState xfer_state = xfer.getState(); System.out.println(": " + xfer_state); }
public static void waitForCompletion(Transfer xfer) { try { xfer.waitForCompletion(); } catch (AmazonServiceException e) { System.err.println("Amazon service error: " + e.getMessage()); System.exit(1); } catch (AmazonClientException e) { System.err.println("Amazon client error: " + e.getMessage()); System.exit(1); } catch (InterruptedException e) { System.err.println("Transfer interrupted: " + e.getMessage()); System.exit(1); } }
getLog().debug("Transferring " + transfer.getProgress().getTotalBytesToTransfer() + " bytes..."); transfer.waitForCompletion(); getLog().info("Transferred " + transfer.getProgress().getBytesTransferred() + " bytes."); } catch (InterruptedException e) { return false; return transfer.getState() == Transfer.TransferState.Completed;
private long doCopy(Mapper.Context context, FileStatus sourceFileStatus, S3UploadDescriptor uploadDescriptor) throws IOException { LOG.debug("Copying {} to {}", sourceFileStatus.getPath(), uploadDescriptor.getTargetPath()); final Path sourcePath = sourceFileStatus.getPath(); Transfer transfer = startTransfer(context, uploadDescriptor); transfer.addProgressListener(new UploadProgressListener(context, description)); try { AmazonClientException e = transfer.waitForException(); if (e != null) { throw new IOException(e); } } catch (InterruptedException e) { throw new RuntimeException("Unable to upload file " + sourcePath, e); } return transfer.getProgress().getBytesTransferred(); }
@Override public synchronized void progressChanged(ProgressEvent progressEvent) { ProgressEventType eventType = progressEvent.getEventType(); if (previousType != eventType) { log.debug("Upload progress event (%s/%s): %s", host, key, eventType); previousType = eventType; } double transferred = transfer.getProgress().getPercentTransferred(); if (transferred >= (previousTransferred + 10.0)) { log.debug("Upload percentage (%s/%s): %.0f%%", host, key, transferred); previousTransferred = transferred; } } };
/** * Set the state based on the states of all file downloads. Assumes all file * downloads are done. * <p> * A single failed sub-transfer makes the entire transfer failed. If there * are no failed sub-transfers, a single canceled sub-transfer makes the * entire transfer canceled. Otherwise, we consider ourselves Completed. */ public void collateFinalState() { boolean seenCanceled = false; for ( T download : subTransfers ) { if ( download.getState() == TransferState.Failed ) { setState(TransferState.Failed); return; } else if ( download.getState() == TransferState.Canceled ) { seenCanceled = true; } } if ( seenCanceled ) setState(TransferState.Canceled); else setState(TransferState.Completed); }
@Override public synchronized boolean isDone() { for ( Transfer subTransfer : subTransfers ) { if ( !subTransfer.isDone() ) return false; } return true; }
AmazonClientException amazonClientException = transfer.waitForException(); if (amazonClientException != null) { throw amazonClientException;
private long doCopy(Mapper.Context context, FileStatus sourceFileStatus, S3UploadDescriptor uploadDescriptor) throws IOException { LOG.debug("Copying {} to {}", sourceFileStatus.getPath(), uploadDescriptor.getTargetPath()); final Path sourcePath = sourceFileStatus.getPath(); Transfer transfer = startTransfer(context, uploadDescriptor); transfer.addProgressListener(new UploadProgressListener(context, description)); try { AmazonClientException e = transfer.waitForException(); if (e != null) { throw new IOException(e); } } catch (InterruptedException e) { throw new RuntimeException("Unable to upload file " + sourcePath, e); } return transfer.getProgress().getBytesTransferred(); }
@Override public void transferStateChanged(Transfer transfer, TransferState state) { if (state == TransferState.Completed) { // NOTE: running progress doesn't seem to be reported correctly. // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment // so just printing total bytes when completed. LOG .debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey, transfer.getProgress().getTotalBytesToTransfer()); } } }
/** * Set the state based on the states of all file downloads. Assumes all file * downloads are done. * <p> * A single failed sub-transfer makes the entire transfer failed. If there * are no failed sub-transfers, a single canceled sub-transfer makes the * entire transfer canceled. Otherwise, we consider ourselves Completed. */ public void collateFinalState() { boolean seenCanceled = false; for ( T download : subTransfers ) { if ( download.getState() == TransferState.Failed ) { setState(TransferState.Failed); return; } else if ( download.getState() == TransferState.Canceled ) { seenCanceled = true; } } if ( seenCanceled ) setState(TransferState.Canceled); else setState(TransferState.Completed); }
@Override public synchronized boolean isDone() { for ( Transfer subTransfer : subTransfers ) { if ( !subTransfer.isDone() ) return false; } return true; }
AmazonClientException amazonClientException = transfer.waitForException(); if (amazonClientException != null) { throw amazonClientException;
TransferProgress transferProgress = transfer.getProgress(); if (transfer.isDone() || stepCount % 300 == 0) while (!transfer.isDone()); TransferState transferState = transfer.getState(); if (transferState == TransferState.Failed) AmazonClientException amazonClientException = transfer.waitForException(); throw new IllegalStateException("The transfer operation \"" + transfer.getDescription() + "\" failed for an unknown reason."); "The transfer operation \"" + transfer.getDescription() + "\" did not complete successfully. Current state: \"" + transferState + "\"."); results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred()); results.setTotalFilesTransferred(1L);
@Override public void transferStateChanged(Transfer transfer, TransferState state) { if (state == TransferState.Completed) { // NOTE: running progress doesn't seem to be reported correctly. // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment // so just printing total bytes when completed. LOG .debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey, transfer.getProgress().getTotalBytesToTransfer()); } } }
private void doItForJob(String jobId) throws InterruptedException { logger.info("Beginning download of "+jobId); String bucketName= getLast(on("/").omitEmptyStrings().split(awsLogUri)); Transfer that=transferManager.downloadDirectory(bucketName, jobId, new File(localLogTarget)); that.waitForCompletion(); } }
TransferProgress transferProgress = transfer.getProgress(); if (transfer.isDone() || stepCount % 300 == 0) while (!transfer.isDone()); TransferState transferState = transfer.getState(); if (transferState == TransferState.Failed) AmazonClientException amazonClientException = transfer.waitForException(); throw new IllegalStateException("The transfer operation \"" + transfer.getDescription() + "\" failed for an unknown reason."); "The transfer operation \"" + transfer.getDescription() + "\" did not complete successfully. Current state: \"" + transferState + "\"."); results.setTotalBytesTransferred(transfer.getProgress().getBytesTransferred()); results.setTotalFilesTransferred(1L);
@Override public synchronized void progressChanged(ProgressEvent progressEvent) { ProgressEventType eventType = progressEvent.getEventType(); if (previousType != eventType) { log.debug("Upload progress event (%s/%s): %s", host, key, eventType); previousType = eventType; } double transferred = transfer.getProgress().getPercentTransferred(); if (transferred >= (previousTransferred + 10.0)) { log.debug("Upload percentage (%s/%s): %.0f%%", host, key, transferred); previousTransferred = transferred; } } };
@Override public synchronized void progressChanged(ProgressEvent progressEvent) { ProgressEventType eventType = progressEvent.getEventType(); if (previousType != eventType) { log.debug("Upload progress event (%s/%s): %s", host, key, eventType); previousType = eventType; } double transferred = transfer.getProgress().getPercentTransferred(); if (transferred >= (previousTransferred + 10.0)) { log.debug("Upload percentage (%s/%s): %.0f%%", host, key, transferred); previousTransferred = transferred; } } };