@Override NativeFileSystemStore getNativeFileSystemStore() throws IOException { return new Jets3tNativeFileSystemStore(); } }
@Override public PartialListing list(String prefix, int maxListingLength) throws IOException { return list(prefix, maxListingLength, null, false); }
/** * Handle any service exception by translating it into an IOException * @param e exception * @param key key sought from object store * @throws IOException exception -always */ private void handleException(Exception e, String key) throws IOException { throw processException(e, e, key); }
@Override public void delete(String key) throws IOException { try { LOG.debug("Deleting key: {} from bucket: {}", key, bucket.getName()); s3Service.deleteObject(bucket, key); } catch (ServiceException e) { handleException(e, key); } }
@Override public void storeFile(String key, File file, byte[] md5Hash) throws IOException { if (multipartEnabled && file.length() >= multipartBlockSize) { storeLargeFile(key, file, md5Hash); return; } BufferedInputStream in = null; try { in = new BufferedInputStream(new FileInputStream(file)); S3Object object = new S3Object(key); object.setDataInputStream(in); object.setContentType("binary/octet-stream"); object.setContentLength(file.length()); object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); if (md5Hash != null) { object.setMd5Hash(md5Hash); } s3Service.putObject(bucket, object); } catch (ServiceException e) { handleException(e, key); } finally { IOUtils.closeStream(in); } }
@Override public void copy(String srcKey, String dstKey) throws IOException { try { if(LOG.isDebugEnabled()) { LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName()); } if (multipartEnabled) { S3Object object = s3Service.getObjectDetails(bucket, srcKey, null, null, null, null); if (multipartCopyBlockSize > 0 && object.getContentLength() > multipartCopyBlockSize) { copyLargeFile(object, dstKey); return; } } S3Object dstObject = new S3Object(dstKey); dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), dstObject, false); } catch (ServiceException e) { handleException(e, srcKey); } }
public void initialize(URI uri, Configuration conf) throws IOException { S3Credentials s3Credentials = new S3Credentials(); s3Credentials.initialize(uri, conf); try { AWSCredentials awsCredentials = new AWSCredentials(s3Credentials.getAccessKey(), s3Credentials.getSecretAccessKey()); this.s3Service = new RestS3Service(awsCredentials); } catch (S3ServiceException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } throw new S3Exception(e); } bucket = new S3Bucket(uri.getHost()); createBucket(bucket.getName()); }
@Override public void dump() throws IOException { StringBuilder sb = new StringBuilder("S3 Native Filesystem, "); sb.append(bucket.getName()).append("\n"); try { S3Object[] objects = s3Service.listObjects(bucket.getName()); for (S3Object object : objects) { sb.append(object.getKey()).append("\n"); } } catch (S3ServiceException e) { handleException(e); } System.out.println(sb); }
@Override public PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recurse) throws IOException { return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey); }
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); methodNameToPolicyMap.put("rename", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
public void rename(String srcKey, String dstKey) throws IOException { try { s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey)); } catch (ServiceException e) { handleException(e, srcKey); } }
/** * Handle any service exception by translating it into an IOException * @param e exception * @throws IOException exception -always */ private void handleException(Exception e) throws IOException { throw processException(e, e, ""); } /**
public PartialListing list(String prefix, int maxListingLength) throws IOException { return list(prefix, maxListingLength, null); }
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
/** * * @param key * The key is the object name that is being retrieved from the S3 bucket * @return * This method returns null if the key is not found * @throws IOException */ @Override public InputStream retrieve(String key, long byteRangeStart) throws IOException { try { LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}", key, bucket.getName(), byteRangeStart); S3Object object = s3Service.getObject(bucket, key, null, null, null, null, byteRangeStart, null); return object.getDataInputStream(); } catch (ServiceException e) { handleException(e, key); return null; } }
if (thrown.getCause() != null) { result = processException(thrown.getCause(), original, key); } else if (thrown instanceof HttpException) {
public PartialListing list(String prefix, int maxListingLength, String priorLastKey) throws IOException { return list(prefix, PATH_DELIMITER, maxListingLength, priorLastKey); }
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
@Override public void initialize(URI uri, Configuration conf) throws IOException { S3Credentials s3Credentials = new S3Credentials(); s3Credentials.initialize(uri, conf); try { AWSCredentials awsCredentials = new AWSCredentials(s3Credentials.getAccessKey(), s3Credentials.getSecretAccessKey()); this.s3Service = new RestS3Service(awsCredentials); } catch (S3ServiceException e) { handleException(e); } multipartEnabled = conf.getBoolean("fs.s3n.multipart.uploads.enabled", false); multipartBlockSize = Math.min( conf.getLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024), MAX_PART_SIZE); multipartCopyBlockSize = Math.min( conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE), MAX_PART_SIZE); serverSideEncryptionAlgorithm = conf.get("fs.s3n.server-side-encryption-algorithm"); bucket = new S3Bucket(uri.getHost()); }
public PartialListing list(String prefix, int maxListingLength) throws IOException { return list(prefix, maxListingLength, null); }