/** * Returns an InputStream only when the status is DOWNLOAD_FINISHED. * * The caller of this method must close the InputStream to prevent resource leaks! * * @return S3ObjectInputStream of the object. */ public InputStream getS3ObjectInputStream() { // Check if the download is finished if (status != Status.DOWNLOAD_FINISHED) { return null; } return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key); }
@Override public DataStoreTO getStoreTO(DataStore store) { ImageStoreImpl imgStore = (ImageStoreImpl)store; Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId()); return new S3TO(imgStore.getId(), imgStore.getUuid(), details.get(ApiConstants.S3_ACCESS_KEY), details.get(ApiConstants.S3_SECRET_KEY), details.get(ApiConstants.S3_END_POINT), details.get(ApiConstants.S3_BUCKET_NAME), details.get(ApiConstants.S3_SIGNER), details.get(ApiConstants.S3_HTTPS_FLAG) == null ? false : Boolean.parseBoolean(details.get(ApiConstants.S3_HTTPS_FLAG)), details.get(ApiConstants.S3_CONNECTION_TIMEOUT) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_CONNECTION_TIMEOUT)), details.get(ApiConstants.S3_MAX_ERROR_RETRY) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_MAX_ERROR_RETRY)), details.get(ApiConstants.S3_SOCKET_TIMEOUT) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(), _configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao.getValue(Config.S3EnableRRS.toString())), getMaxSingleUploadSizeInBytes(), details.get(ApiConstants.S3_CONNECTION_TTL) == null ? null : Integer.valueOf(details.get(ApiConstants.S3_CONNECTION_TTL)), details.get(ApiConstants.S3_USE_TCP_KEEPALIVE) == null ? null : Boolean.parseBoolean(details.get(ApiConstants.S3_USE_TCP_KEEPALIVE))); }
protected String backupSnapshotToS3(final Connection connection, final S3TO s3, final String srUuid, final String folder, final String snapshotUuid, final Boolean iSCSIFlag, final int wait) { final String filename = iSCSIFlag ? "VHD-" + snapshotUuid : snapshotUuid + ".vhd"; final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-" : "/var/run/sr-mount/") + srUuid; final String key = folder + "/" + filename; // String.format("/snapshots/%1$s", snapshotUuid); try { final List<String> parameters = newArrayList(flattenProperties(s3, ClientOptions.class)); // https workaround for Introspector bug that does not // recognize Boolean accessor methods ... parameters.addAll(Arrays.asList("operation", "put", "filename", dir + "/" + filename, "iSCSIFlag", iSCSIFlag.toString(), "bucket", s3.getBucketName(), "key", key, "https", s3.isHttps() != null ? s3.isHttps().toString() : "null", "maxSingleUploadSizeInBytes", String.valueOf(s3.getMaxSingleUploadSizeInBytes()))); final String result = hypervisorResource.callHostPluginAsync(connection, "s3xenserver", "s3", wait, parameters.toArray(new String[parameters.size()])); if (result != null && result.equals("true")) { return key; } return null; } catch (final Exception e) { s_logger.error(String.format("S3 upload failed of snapshot %1$s due to %2$s.", snapshotUuid, e.toString()), e); } return null; }
LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes"); PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata); if (s3TO.isEnableRRS()) { putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy); LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!"); } else { LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
public void cleanupAfterError() { LOGGER.warn("Cleanup after error, trying to remove object: " + s3Key); S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key); }
protected String copyToS3(final File srcFile, final S3TO destStore, final String destPath) throws InterruptedException { final String key = destPath + S3Utils.SEPARATOR + srcFile.getName(); putFile(destStore, srcFile, destStore.getBucketName(), key).waitForCompletion(); return key; }
@Override public String createEntityExtractUrl(DataStore store, String key, ImageFormat format, DataObject dataObject) { /** * Generate a pre-signed URL for the given object. */ S3TO s3 = (S3TO)getStoreTO(store); if(s_logger.isDebugEnabled()) { s_logger.debug("Generating pre-signed s3 entity extraction URL for object: " + key); } Date expiration = new Date(); long milliSeconds = expiration.getTime(); // Get extract url expiration interval set in global configuration (in seconds) String urlExpirationInterval = _configDao.getValue(Config.ExtractURLExpirationInterval.toString()); // Expired after configured interval (in milliseconds), default 14400 seconds milliSeconds += 1000 * NumbersUtil.parseInt(urlExpirationInterval, 14400); expiration.setTime(milliSeconds); URL s3url = S3Utils.generatePresignedUrl(s3, s3.getBucketName(), key, expiration); s_logger.info("Pre-Signed URL = " + s3url.toString()); return s3url.toString(); } }
@Override public boolean stopDownload() { switch (status) { case IN_PROGRESS: if (getMethod != null) { getMethod.abort(); } break; case UNKNOWN: case NOT_STARTED: case RECOVERABLE_ERROR: case UNRECOVERABLE_ERROR: case ABORTED: case DOWNLOAD_FINISHED: // Remove the object if it already has been uploaded. S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key); break; default: break; } status = TemplateDownloader.Status.ABORTED; return true; }
@Override public void progressChanged(ProgressEvent progressEvent) { // Record the amount of bytes transferred. totalBytes += progressEvent.getBytesTransferred(); LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds"); if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) { status = Status.IN_PROGRESS; } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { status = Status.DOWNLOAD_FINISHED; } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) { status = Status.ABORTED; } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) { status = Status.UNRECOVERABLE_ERROR; } } });
Map<Long, TemplateProp> s3ListVolume(S3TO s3) { String bucket = s3.getBucketName(); // List the objects in the source directory on S3 final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, VOLUME_ROOT_DIR); if (objectSummaries == null) { return null; } Map<Long, TemplateProp> tmpltInfos = new HashMap<Long, TemplateProp>(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); // String installPath = StringUtils.substringBeforeLast(key, // S3Utils.SEPARATOR); Long id = determineS3VolumeIdFromKey(key); // TODO: how to get volume template name TemplateProp tInfo = new TemplateProp(id.toString(), key, objectSummary.getSize(), objectSummary.getSize(), true, false); tmpltInfos.put(id, tInfo); } return tmpltInfos; }
protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3, DataTO destData, NfsTO destImageStore) { final String storagePath = destImageStore.getUrl(); final String destPath = destData.getPath(); try { String downloadPath = determineStorageTemplatePath(storagePath, destPath, _nfsVersion); final File downloadDirectory = _storage.getFile(downloadPath); if (downloadDirectory.exists()) { s_logger.debug("Directory " + downloadPath + " already exists"); } else { if (!downloadDirectory.mkdirs()) { final String errMsg = "Unable to create directory " + downloadPath + " to copy from S3 to cache."; s_logger.error(errMsg); return new CopyCmdAnswer(errMsg); } } File destFile = new File(downloadDirectory, substringAfterLast(srcData.getPath(), S3Utils.SEPARATOR)); S3Utils.getFile(s3, s3.getBucketName(), srcData.getPath(), destFile).waitForCompletion(); return postProcessing(destFile, downloadPath, destPath, srcData, destData); } catch (Exception e) { final String errMsg = format("Failed to download" + "due to $1%s", e.getMessage()); s_logger.error(errMsg, e); return new CopyCmdAnswer(errMsg); } }
Map<String, TemplateProp> s3ListTemplate(S3TO s3) { String bucket = s3.getBucketName(); // List the objects in the source directory on S3 final List<S3ObjectSummary> objectSummaries = S3Utils.listDirectory(s3, bucket, TEMPLATE_ROOT_DIR); if (objectSummaries == null) { return null; } Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>(); for (S3ObjectSummary objectSummary : objectSummaries) { String key = objectSummary.getKey(); // String installPath = StringUtils.substringBeforeLast(key, // S3Utils.SEPARATOR); String uniqueName = determineS3TemplateNameFromKey(key); // TODO: isPublic value, where to get? TemplateProp tInfo = new TemplateProp(uniqueName, key, objectSummary.getSize(), objectSummary.getSize(), true, false); tmpltInfos.put(uniqueName, tInfo); } return tmpltInfos; }
final S3TO s3 = (S3TO)dstore; final String path = obj.getPath(); final String bucket = s3.getBucketName(); try { S3Utils.deleteDirectory(s3, bucket, path);
final S3TO s3 = (S3TO)dstore; final String path = obj.getPath(); final String bucket = s3.getBucketName(); try { S3Utils.deleteDirectory(s3, bucket, path);
final S3TO s3 = (S3TO)dstore; final String path = cmd.getDirectory(); final String bucket = s3.getBucketName(); try { S3Utils.deleteDirectory(s3, bucket, path);
final S3TO s3 = (S3TO)dstore; final String path = obj.getPath(); final String bucket = s3.getBucketName(); try { S3Utils.deleteObject(s3, bucket, path);
private Answer execute(ListTemplateCommand cmd) { if (!_inSystemVM) { return new ListTemplateAnswer(null, null); } DataStoreTO store = cmd.getDataStore(); if (store instanceof NfsTO) { NfsTO nfs = (NfsTO)store; String secUrl = nfs.getUrl(); String root = getRootDir(secUrl, cmd.getNfsVersion()); Map<String, TemplateProp> templateInfos = _dlMgr.gatherTemplateInfo(root); return new ListTemplateAnswer(secUrl, templateInfos); } else if (store instanceof SwiftTO) { SwiftTO swift = (SwiftTO)store; Map<String, TemplateProp> templateInfos = swiftListTemplate(swift); return new ListTemplateAnswer(swift.toString(), templateInfos); } else if (store instanceof S3TO) { S3TO s3 = (S3TO)store; Map<String, TemplateProp> templateInfos = s3ListTemplate(s3); return new ListTemplateAnswer(s3.getBucketName(), templateInfos); } else { return new Answer(cmd, false, "Unsupported image data store: " + store); } }
final String bucket = s3.getBucketName(); File srcFile = findFile(templatePath); if (srcFile == null) {
private Answer execute(ListVolumeCommand cmd) { if (!_inSystemVM) { return new ListVolumeAnswer(cmd.getSecUrl(), null); } DataStoreTO store = cmd.getDataStore(); if (store instanceof NfsTO) { String root = getRootDir(cmd.getSecUrl(), _nfsVersion); Map<Long, TemplateProp> templateInfos = _dlMgr.gatherVolumeInfo(root); return new ListVolumeAnswer(cmd.getSecUrl(), templateInfos); } else if (store instanceof S3TO) { S3TO s3 = (S3TO)store; Map<Long, TemplateProp> templateInfos = s3ListVolume(s3); return new ListVolumeAnswer(s3.getBucketName(), templateInfos); } else { return new Answer(cmd, false, "Unsupported image data store: " + store); } }