@Override protected boolean deleteObject(String key) throws IOException { try { mClient.deleteObject(mBucketName, key); } catch (ServiceException e) { LOG.error("Failed to delete {}", key, e); return false; } return true; }
private StorageObjectsChunk getObjectListingChunk(String key, String delimiter, String priorLastKey) { StorageObjectsChunk res; try { res = mClient.listObjectsChunked(mBucketName, key, delimiter, getListingChunkLength(mAlluxioConf), priorLastKey); } catch (ServiceException e) { LOG.error("Failed to list path {}", key, e); res = null; } return res; }
/** * Creates a new instance of {@link GCSInputStream}, at a specific position. * * @param bucketName the name of the bucket * @param key the key of the file * @param client the client for GCS * @param pos the position to start * @throws ServiceException if a service exception occurs */ GCSInputStream(String bucketName, String key, GoogleStorageService client, long pos) throws ServiceException { mBucketName = bucketName; mKey = key; mClient = client; mPos = pos; // For an empty file setting start pos = 0 will throw a ServiceException if (mPos > 0) { mObject = mClient.getObject(mBucketName, mKey, null, null, null, null, mPos, null); } else { mObject = mClient.getObject(mBucketName, mKey); } mInputStream = new BufferedInputStream(mObject.getDataInputStream()); }
S3Service s3Service = new RestS3Service(awsCredentials); String key = formatLatLon(x, y) + ".tiff"; S3Object object = s3Service.getObject(awsBucketName, key);
GoogleStorageService googleStorageService = new GoogleStorageService(googleCredentials); StorageOwner storageOwner = googleStorageService.getAccountOwner(); String accountOwnerId = "unknown"; String accountOwner = "unknown"; GSAccessControlList acl = googleStorageService.getBucketAcl(bucketName); short bucketMode = GCSUtils.translateBucketAcl(acl, accountOwnerId);
@Override protected boolean createEmptyObject(String key) { try { GSObject obj = new GSObject(key); obj.setDataInputStream(new ByteArrayInputStream(new byte[0])); obj.setContentLength(0); obj.setMd5Hash(DIR_HASH); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); mClient.putObject(mBucketName, obj); return true; } catch (ServiceException e) { LOG.error("Failed to create directory: {}", key, e); return false; } }
@Override protected HttpUriRequest setupConnection(HTTP_METHOD method, String bucketName, String objectKey, Map<String, String> requestParameters) throws ServiceException { final HttpUriRequest request = super.setupConnection(method, bucketName, objectKey, requestParameters); // Use API version 2 if we are using OAuth2 credentials if (getProviderCredentials() instanceof OAuth2Credentials) { request.setHeader("x-goog-api-version", "2"); } return request; }
@Override protected boolean copyObject(String src, String dst) { LOG.debug("Copying {} to {}", src, dst); GSObject obj = new GSObject(dst); // Retry copy for a few times, in case some Jets3t or GCS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { mClient.copyObject(mBucketName, src, mBucketName, obj, false); return true; } catch (ServiceException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
@Override protected ObjectStatus getObjectStatus(String key) { try { GSObject meta = mClient.getObjectDetails(mBucketName, key); if (meta == null) { return null; } return new ObjectStatus(key, meta.getMd5HashAsBase64(), meta.getContentLength(), meta.getLastModifiedDate().getTime()); } catch (ServiceException e) { return null; } }
@Override public void putObjectWithRequestEntityImpl(String bucketName, StorageObject object, HttpEntity requestEntity, Map<String, String> requestParams) throws ServiceException { super.putObjectWithRequestEntityImpl(bucketName, object, requestEntity, requestParams); }
@Override protected StorageObjectsChunk listObjectsChunkedImpl(String bucketName, String prefix, String delimiter, long maxListingLength, String priorLastKey, boolean completeListing) throws ServiceException { return listObjectsInternal(bucketName, prefix, delimiter, maxListingLength, completeListing, priorLastKey); }
@Override public GSObject getObject(String bucketName, String objectKey, Calendar ifModifiedSince, Calendar ifUnmodifiedSince, String[] ifMatchTags, String[] ifNoneMatchTags, Long byteRangeStart, Long byteRangeEnd) throws ServiceException { return (GSObject) super.getObject(bucketName, objectKey, ifModifiedSince, ifUnmodifiedSince, ifMatchTags, ifNoneMatchTags, byteRangeStart, byteRangeEnd); }
protected MultipartUpload multipartStartUploadImpl(String bucketName, String objectKey, Map<String, Object> metadataProvided, AccessControlList acl, String storageClass) throws S3ServiceException { return this.multipartStartUploadImpl( bucketName, objectKey, metadataProvided, acl, storageClass, null); }
/** * @return * If true, JetS3t will enable support for Server-Side Encryption. Only enabled for * Amazon S3 end-point by default. */ @Override protected boolean getEnableServerSideEncryption() { return ! isTargettingGoogleStorageService(); }
@Override protected VersionOrDeleteMarkersChunk listVersionedObjectsChunkedImpl(String bucketName, String prefix, String delimiter, long maxListingLength, String priorLastKey, String priorLastVersion, boolean completeListing) throws S3ServiceException { return listVersionedObjectsInternal(bucketName, prefix, delimiter, maxListingLength, completeListing, priorLastKey, priorLastVersion); }
public GSBucket createBucket(String bucketName, String location, AccessControlList acl) throws ServiceException { return createBucket(bucketName, location, acl, null); }
public GSObject putObject(String bucketName, GSObject object) throws ServiceException { return (GSObject) super.putObject(bucketName, object); }
@Override public WebsiteConfig getWebsiteConfigImpl(String bucketName) throws ServiceException { return super.getWebsiteConfigImpl(bucketName); }
@Override public void close() throws IOException { if (mClosed.getAndSet(true)) { return; } mLocalOutputStream.close(); try { GSObject obj = new GSObject(mKey); obj.setBucketName(mBucketName); obj.setDataInputFile(mFile); obj.setContentLength(mFile.length()); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); if (mHash != null) { obj.setMd5Hash(mHash.digest()); } else { LOG.warn("MD5 was not computed for: {}", mKey); } mClient.putObject(mBucketName, obj); if (!mFile.delete()) { LOG.error("Failed to delete temporary file @ {}", mFile.getPath()); } } catch (ServiceException e) { LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath()); throw new IOException(e); } } }
/** * This method leverages the ability to open a stream from GCS from a given offset. When the * underlying stream has fewer bytes buffered than the skip request, the stream is closed, and * a new stream is opened starting at the requested offset. * * @param n number of bytes to skip * @return the number of bytes skipped */ @Override public long skip(long n) throws IOException { if (mInputStream.available() >= n) { return mInputStream.skip(n); } // The number of bytes to skip is possibly large, open a new stream from GCS. mInputStream.close(); mPos += n; try { mObject = mClient.getObject(mBucketName, mKey, null /* ignore ModifiedSince */, null /* ignore UnmodifiedSince */, null /* ignore MatchTags */, null /* ignore NoneMatchTags */, mPos /* byteRangeStart */, null /* ignore byteRangeEnd */); mInputStream = new BufferedInputStream(mObject.getDataInputStream()); } catch (ServiceException e) { throw new IOException(e); } return n; } }