@JsonIgnore public BackupRestoreContext withLocalLocation(final String localLocation){ return create( nodeId, name, externalLocation, localLocation, accountId, secretKey, usesEmc, restoreType); } }
@Override public int hashCode() { return Objects.hash(getNodeId(), getName(), getExternalLocation(), getLocalLocation(), getAccountId(), getSecretKey(), getRestoreType()); }
private TransferManager getS3TransferManager(BackupRestoreContext ctx) { final String accessKey = ctx.getAccountId(); final String secretKey = ctx.getSecretKey(); final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey); TransferManager tx = new TransferManager(basicAWSCredentials); return tx; }
String getPrefixKey(BackupRestoreContext ctx) throws URISyntaxException { URI uri = new URI(ctx.getExternalLocation()); String[] segments = uri.getPath().split("/"); int startIndex = uri.getScheme().equals(AmazonS3Client.S3_SERVICE_NAME) ? 1 : 2; String prefixKey = ""; for (int i=startIndex; i<segments.length; i++) { prefixKey += segments[i]; if (i < segments.length - 1) { prefixKey += "/"; } } prefixKey = (prefixKey.length() > 0 && !prefixKey.endsWith("/")) ? prefixKey + "/" : prefixKey; prefixKey += ctx.getName(); // append backup name return prefixKey; }
@Override public void download(BackupRestoreContext ctx) throws Exception { final String backupName = ctx.getName(); final String nodeId = ctx.getNodeId(); final File[] keyspaces = getNonSystemKeyspaces(ctx); final String bucketName = getBucketName(ctx); final String localLocation = ctx.getLocalLocation(); final TransferManager tx = getS3TransferManager(ctx); final AmazonS3Client amazonS3Client = getAmazonS3Client(ctx); if (Objects.equals(ctx.getRestoreType(), new String("new"))) { final Map<String, Long> snapshotFileKeys = listSnapshotFiles(amazonS3Client, bucketName,
@Override public void upload(BackupRestoreContext ctx) throws Exception { final String localLocation = ctx.getLocalLocation(); final String backupName = ctx.getName(); final String nodeId = ctx.getNodeId(); final String key = getPrefixKey(ctx) + "/" + nodeId; LOGGER.info("Backup key: " + key);
private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException { final String accessKey = ctx.getAccountId(); final String secretKey = ctx.getSecretKey(); String endpoint = getEndpoint(ctx); LOGGER.info("endpoint: {}", endpoint); final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey); final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials); amazonS3Client.setEndpoint(endpoint); if (ctx.usesEmc()) { final S3ClientOptions options = new S3ClientOptions(); options.setPathStyleAccess(true); amazonS3Client.setS3ClientOptions(options); } return amazonS3Client; }
@Override public void run() { try { LOGGER.info("Starting DownloadSnapshot task using context: {}", context); // Send TASK_RUNNING sendStatus(driver, Protos.TaskState.TASK_RUNNING, "Started downloading snapshot"); // cleanup downloaded snapshot directory recursively if exists. Path rootPath = Paths.get(context.getLocalLocation() + File.separator + context.getName()); if (rootPath.toFile().exists()) { Files.walk(rootPath, FileVisitOption.FOLLOW_LINKS) .sorted(Comparator.reverseOrder()) .map(Path::toFile) .forEach(File::delete); } backupStorageDriver.download(context); // Send TASK_FINISHED sendStatus(driver, Protos.TaskState.TASK_FINISHED, "Finished downloading snapshots"); } catch (Throwable t) { LOGGER.error("Download snapshot failed",t); sendStatus(driver, Protos.TaskState.TASK_FAILED, t.getMessage()); } }
String getBucketName(BackupRestoreContext ctx) throws URISyntaxException { URI uri = new URI(ctx.getExternalLocation()); LOGGER.info("URI: " + uri); if (uri.getScheme().equals(AmazonS3Client.S3_SERVICE_NAME)) { return uri.getHost(); } else { return uri.getPath().split("/")[1]; } }
@Override public String downloadSchema(BackupRestoreContext ctx) throws Exception { final String nodeId = ctx.getNodeId(); final AmazonS3Client amazonS3Client = getAmazonS3Client(ctx); final String key = getPrefixKey(ctx) + "/" + nodeId + "/" + StorageUtil.SCHEMA_FILE; S3Object object = amazonS3Client.getObject( new GetObjectRequest(getBucketName(ctx), key)); InputStream objectData = object.getObjectContent(); String schema = IOUtils.toString(objectData, "UTF-8"); objectData.close(); return schema; }
@Override public void run() { try { // Send TASK_RUNNING sendStatus(driver, Protos.TaskState.TASK_RUNNING, "Started taking snapshot"); final String snapshotName = this.cassandraTask.getBackupRestoreContext().getName(); final List<String> nonSystemKeyspaces = daemon.getNonSystemKeySpaces(); LOGGER.info("Started taking snapshot for non system keyspaces: {}", nonSystemKeyspaces); for (String keyspace : nonSystemKeyspaces) { LOGGER.info("Clearing snapshot {} for keyspace: {}", snapshotName, keyspace); daemon.clearSnapshot(snapshotName, keyspace); LOGGER.info("Taking snapshot {} for keyspace: {}", snapshotName, keyspace); daemon.takeSnapShot(snapshotName, keyspace); } // Send TASK_FINISHED sendStatus(driver, Protos.TaskState.TASK_FINISHED, "Finished taking snapshot for non system keyspaces: " + nonSystemKeyspaces); } catch (Throwable t) { LOGGER.error("Snapshot failed",t); sendStatus(driver, Protos.TaskState.TASK_FAILED, t.getMessage()); } }
private File[] getNonSystemKeyspaces(BackupRestoreContext ctx) { File file = new File(ctx.getLocalLocation()); File[] directories = file.listFiles( (current, name) -> new File(current, name).isDirectory() && name.compareTo("system") != 0); return directories; }
"Started restoring snapshot"); if (Objects.equals(context.getRestoreType(), new String("new"))) { final String keyspaceDirectory = context.getLocalLocation() + File.separator + context.getName() + File.separator + context.getNodeId(); Path rootPath = Paths.get(context.getLocalLocation() + File.separator + context.getName()); if (rootPath.toFile().exists()) { Files.walk(rootPath, FileVisitOption.FOLLOW_LINKS) final String localLocation = context.getLocalLocation(); final List<String> keyspaces = cassandra.getNonSystemKeySpaces(); for (String keyspace : keyspaces) {
String getEndpoint(BackupRestoreContext ctx) throws URISyntaxException { URI uri = new URI(ctx.getExternalLocation()); String scheme = uri.getScheme(); if (scheme.equals(AmazonS3Client.S3_SERVICE_NAME)) { return Constants.S3_HOSTNAME; } else { String endpoint = scheme + "://" + uri.getHost(); int port = uri.getPort(); if (port != -1) { endpoint += ":" + Integer.toString(port); } return endpoint; } }
@Override public void uploadSchema(BackupRestoreContext ctx, String schema) throws Exception { final String nodeId = ctx.getNodeId(); final AmazonS3Client amazonS3Client = getAmazonS3Client(ctx); final String key = getPrefixKey(ctx) + "/" + nodeId + "/" + StorageUtil.SCHEMA_FILE; final InputStream stream = new ByteArrayInputStream(schema.getBytes(StandardCharsets.UTF_8)); amazonS3Client.putObject(getBucketName(ctx), key, stream, new ObjectMetadata()); }
@Override public void run() { try { // Send TASK_RUNNING sendStatus(driver, Protos.TaskState.TASK_RUNNING, "Started uploading snapshots"); // Upload snapshots to external location. backupStorageDriver.upload(context); // Once we have uploaded all existing snapshots, let's clear on-disk snapshots daemon.clearSnapShot(context.getName()); // Send TASK_FINISHED sendStatus(driver, Protos.TaskState.TASK_FINISHED, "Finished uploading snapshots"); } catch (Throwable t) { LOGGER.error("Upload snapshot failed", t); sendStatus(driver, Protos.TaskState.TASK_FAILED, t.getMessage()); } }
public static final CassandraData createBackupUploadData( final String hostname, final BackupRestoreContext context) { return new CassandraData( CassandraTask.TYPE.BACKUP_UPLOAD, hostname, context.getNodeId(), context.getName(), context.getExternalLocation(), context.getLocalLocation(), context.getAccountId(), context.getSecretKey(), context.getUsesEmc(), context.getRestoreType()); }
@JsonIgnore public BackupRestoreContext forNode(final String nodeId){ return create( nodeId, name, externalLocation, localLocation, accountId, secretKey, usesEmc, restoreType); }
public static BackupStorageDriver createStorageDriver(CassandraTask cassandraTask) { String externalLocation = null; switch (cassandraTask.getType()) { case BACKUP_SNAPSHOT: externalLocation = ((BackupUploadTask)cassandraTask).getBackupRestoreContext().getExternalLocation(); break; case BACKUP_SCHEMA: externalLocation = ((BackupSchemaTask)cassandraTask).getBackupRestoreContext().getExternalLocation(); break; case SCHEMA_RESTORE: externalLocation = ((RestoreSchemaTask)cassandraTask).getBackupRestoreContext().getExternalLocation(); break; case SNAPSHOT_DOWNLOAD: externalLocation = ((DownloadSnapshotTask)cassandraTask).getBackupRestoreContext().getExternalLocation(); break; } return getBackupStorageDriver(externalLocation); }
public static final CassandraData createSnapshotDownloadData( final String hostname, final BackupRestoreContext context) { return new CassandraData( CassandraTask.TYPE.SNAPSHOT_DOWNLOAD, hostname, context.getNodeId(), context.getName(), context.getExternalLocation(), context.getLocalLocation(), context.getAccountId(), context.getSecretKey(), context.getUsesEmc(), context.getRestoreType()); }