@Override public void delete(String key) throws IOException { try { LOG.debug("Deleting key: {} from bucket: {}", key, bucket.getName()); s3Service.deleteObject(bucket, key); } catch (ServiceException e) { handleException(e, key); } }
@Override public void dump() throws IOException { StringBuilder sb = new StringBuilder("S3 Native Filesystem, "); sb.append(bucket.getName()).append("\n"); try { S3Object[] objects = s3Service.listObjects(bucket.getName()); for (S3Object object : objects) { sb.append(object.getKey()).append("\n"); } } catch (S3ServiceException e) { handleException(e); } System.out.println(sb); }
public void rename(String srcKey, String dstKey) throws IOException { try { s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey)); } catch (ServiceException e) { handleException(e, srcKey); } }
@Override public void initialize(URI uri, Configuration conf) throws IOException { S3Credentials s3Credentials = new S3Credentials(); s3Credentials.initialize(uri, conf); try { AWSCredentials awsCredentials = new AWSCredentials(s3Credentials.getAccessKey(), s3Credentials.getSecretAccessKey()); this.s3Service = new RestS3Service(awsCredentials); } catch (S3ServiceException e) { handleException(e); } multipartEnabled = conf.getBoolean("fs.s3n.multipart.uploads.enabled", false); multipartBlockSize = Math.min( conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024), MAX_PART_SIZE); multipartCopyBlockSize = Math.min( conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE), MAX_PART_SIZE); serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm"); bucket = new S3Bucket(uri.getHost()); }
/** * * @param key * The key is the object name that is being retrieved from the S3 bucket * @return * This method returns null if the key is not found * @throws IOException */ @Override public InputStream retrieve(String key, long byteRangeStart) throws IOException { try { LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}", key, bucket.getName(), byteRangeStart); S3Object object = s3Service.getObject(bucket, key, null, null, null, null, byteRangeStart, null); return object.getDataInputStream(); } catch (ServiceException e) { handleException(e, key); return null; } }
/** * @param key * The key is the object name that is being retrieved from the S3 bucket * @return * This method returns null if the key is not found * @throws IOException */ @Override public InputStream retrieve(String key) throws IOException { try { LOG.debug("Getting key: {} from bucket: {}", key, bucket.getName()); S3Object object = s3Service.getObject(bucket.getName(), key); return object.getDataInputStream(); } catch (ServiceException e) { handleException(e, key); return null; //return null if key not found } }
@Override public void storeEmptyFile(String key) throws IOException { try { S3Object object = new S3Object(key); object.setDataInputStream(new ByteArrayInputStream(new byte[0])); object.setContentType("binary/octet-stream"); object.setContentLength(0); object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.putObject(bucket, object); } catch (ServiceException e) { handleException(e, key); } }
@Override public void purge(String prefix) throws IOException { String key = ""; try { S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); for (S3Object object : objects) { key = object.getKey(); s3Service.deleteObject(bucket, key); } } catch (S3ServiceException e) { handleException(e, key); } }
@Override public void storeFile(String key, File file, byte[] md5Hash) throws IOException { if (multipartEnabled && file.length() >= multipartBlockSize) { storeLargeFile(key, file, md5Hash); return; } BufferedInputStream in = null; try { in = new BufferedInputStream(new FileInputStream(file)); S3Object object = new S3Object(key); object.setDataInputStream(in); object.setContentType("binary/octet-stream"); object.setContentLength(file.length()); object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); if (md5Hash != null) { object.setMd5Hash(md5Hash); } s3Service.putObject(bucket, object); } catch (ServiceException e) { handleException(e, key); } finally { IOUtils.closeStream(in); } }
public void storeLargeFile(String key, File file, byte[] md5Hash) throws IOException { S3Object object = new S3Object(key); object.setDataInputFile(file); object.setContentType("binary/octet-stream"); object.setContentLength(file.length()); object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); if (md5Hash != null) { object.setMd5Hash(md5Hash); } List<StorageObject> objectsToUploadAsMultipart = new ArrayList<StorageObject>(); objectsToUploadAsMultipart.add(object); MultipartUtils mpUtils = new MultipartUtils(multipartBlockSize); try { mpUtils.uploadObjects(bucket.getName(), s3Service, objectsToUploadAsMultipart, null); } catch (Exception e) { handleException(e, key); } }
@Override public FileMetadata retrieveMetadata(String key) throws IOException { StorageObject object = null; try { LOG.debug("Getting metadata for key: {} from bucket: {}", key, bucket.getName()); object = s3Service.getObjectDetails(bucket.getName(), key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (ServiceException e) { try { // process handleException(e, key); return null; } catch (FileNotFoundException fnfe) { // and downgrade missing files return null; } } finally { if (object != null) { object.closeDataInputStream(); } } }
@Override public void copy(String srcKey, String dstKey) throws IOException { try { if(LOG.isDebugEnabled()) { LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName()); } if (multipartEnabled) { S3Object object = s3Service.getObjectDetails(bucket, srcKey, null, null, null, null); if (multipartCopyBlockSize > 0 && object.getContentLength() > multipartCopyBlockSize) { copyLargeFile(object, dstKey); return; } } S3Object dstObject = new S3Object(dstKey); dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), dstObject, false); } catch (ServiceException e) { handleException(e, srcKey); } }
s3Service.multipartCompleteUpload(multipartUpload, listedParts); } catch (ServiceException e) { handleException(e, srcObject.getKey());
chunk.getCommonPrefixes()); } catch (ServiceException e) { handleException(e, prefix); return null; // never returned - keep compiler happy