Refine search
@Override protected List<String> deleteObjects(List<String> keys) throws IOException { if (!mAlluxioConf.getBoolean(PropertyKey.UNDERFS_S3A_BULK_DELETE_ENABLED)) { return super.deleteObjects(keys); } Preconditions.checkArgument(keys != null && keys.size() <= getListingChunkLengthMax()); try { List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(); for (String key : keys) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } DeleteObjectsResult deletedObjectsResult = mClient.deleteObjects(new DeleteObjectsRequest(mBucketName).withKeys(keysToDelete)); List<String> deletedObjects = new ArrayList<>(); for (DeleteObjectsResult.DeletedObject deletedObject : deletedObjectsResult .getDeletedObjects()) { deletedObjects.add(deletedObject.getKey()); } return deletedObjects; } catch (AmazonClientException e) { throw new IOException(e); } }
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3BucketName); multiObjectDeleteRequest.setKeys(keysSubList); try LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", keyVersion.getKey(), keyVersion.getVersion());
@Override public void deleteRemoteFiles(List<Path> remotePaths) throws BackupRestoreException { if (remotePaths.isEmpty()) return; try { List<DeleteObjectsRequest.KeyVersion> keys = remotePaths .stream() .map( remotePath -> new DeleteObjectsRequest.KeyVersion( remotePath.toString())) .collect(Collectors.toList()); s3Client.deleteObjects( new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); logger.info("Deleted {} objects from S3", remotePaths.size()); } catch (Exception e) { logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); throw new BackupRestoreException(e + " while trying to delete the objects"); } }
for (S3ObjectSummary so : partition) { String key = so.getKey(); keys.add(new KeyVersion(key)); DeleteObjectsRequest deleteReq = new DeleteObjectsRequest(bucketName); deleteReq.setQuiet(true); deleteReq.setKeys(keys);
copyFile(srcKey, dstKey); statistics.incrementWriteOps(1); keysToDelete.add(new DeleteObjectsRequest.KeyVersion(srcKey)); keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); String newDstKey = dstKey + summary.getKey().substring(srcKey.length()); copyFile(summary.getKey(), newDstKey); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket); deleteRequest.setKeys(keysToDelete); s3.deleteObjects(deleteRequest);
@Test public void test() { DeleteObjectsRequest req = new DeleteObjectsRequest("bucketName"); List<KeyVersion> keys = req.getKeys(); assertTrue(keys.size() == 0); keys.add(new KeyVersion("1")); assertTrue(req.getKeys().size() == 1); } }
/** * Convenience method to specify a set of keys without versions. * * @see DeleteObjectsRequest#withKeys(List) */ public DeleteObjectsRequest withKeys(String... keys) { List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(keys.length); for (String key : keys) { keyVersions.add(new KeyVersion(key)); } setKeys(keyVersions); return this; }
@Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) throws AmazonClientException, AmazonServiceException { maybeFail(); for (DeleteObjectsRequest.KeyVersion keyVersion : deleteObjectsRequest.getKeys()) { registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest.getBucketName()); } return super.deleteObjects(deleteObjectsRequest); }
private void writeKeyVersion(XmlWriter xml, KeyVersion keyVersion) { xml.start("Object"); xml.start("Key").value(keyVersion.getKey()).end(); if (keyVersion.getVersion() != null) { xml.start("VersionId").value(keyVersion.getVersion()).end(); } xml.end(); }
private void writeKeyVersion(XmlWriter xml, KeyVersion keyVersion) { xml.start("Object"); xml.start("Key").value(keyVersion.getKey()).end(); if (keyVersion.getVersion() != null) { xml.start("VersionId").value(keyVersion.getVersion()).end(); } xml.end(); }
private void writeKeyVersion(XmlWriter xml, KeyVersion keyVersion) { xml.start("Object"); xml.start("Key").value(keyVersion.getKey()).end(); if (keyVersion.getVersion() != null) { xml.start("VersionId").value(keyVersion.getVersion()).end(); } xml.end(); }
@Override public KeyVersion apply(long[] loc) { long z = loc[2]; long x = loc[0]; long y = loc[1]; StringBuilder sb = new StringBuilder(coordsPrefix); sb.append(z).append('/').append(x).append('/').append(y).append('.').append(extension); return new KeyVersion(sb.toString()); } }
private void writeKeyVersion(XmlWriter xml, KeyVersion keyVersion) { xml.start("Object"); xml.start("Key").value(keyVersion.getKey()).end(); if (keyVersion.getVersion() != null) { xml.start("VersionId").value(keyVersion.getVersion()).end(); } xml.end(); }
@Override public KeyVersion apply(long[] loc) { long z = loc[2]; long x = loc[0]; long y = loc[1]; StringBuilder sb = new StringBuilder(coordsPrefix); sb.append(z).append('/').append(x).append('/').append(y).append('.').append(extension); return new KeyVersion(sb.toString()); } }
@Override public KeyVersion apply(S3ObjectSummary input) { KeyVersion k = new KeyVersion(input.getKey()); return k; } });
/** * Delete mock parent directories which are no longer needed. * Retry policy: retrying; exceptions swallowed. * @param path path */ @Retries.RetryExceptionsSwallowed private void deleteUnnecessaryFakeDirectories(Path path) { List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>(); while (!path.isRoot()) { String key = pathToKey(path); key = (key.endsWith("/")) ? key : (key + "/"); LOG.trace("To delete unnecessary fake directory {} for {}", key, path); keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key)); path = path.getParent(); } try { removeKeys(keysToRemove, false, true); } catch(AmazonClientException | IOException e) { instrumentation.errorIgnored(); if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) { sb.append(kv.getKey()).append(","); } LOG.debug("While deleting keys {} ", sb.toString(), e); } } }
for (File file : params.getFiles()) keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
for (File file : params.getFiles()) keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
for (S3VersionSummary s3VersionSummary : s3VersionSummaries) keyVersions.add(new DeleteObjectsRequest.KeyVersion(s3VersionSummary.getKey(), s3VersionSummary.getVersionId()));
for (S3VersionSummary s3VersionSummary : s3VersionSummaries) keyVersions.add(new DeleteObjectsRequest.KeyVersion(s3VersionSummary.getKey(), s3VersionSummary.getVersionId()));