GoogleStorageService googleStorageService = new GoogleStorageService(googleCredentials); StorageOwner storageOwner = googleStorageService.getAccountOwner(); String accountOwnerId = "unknown"; String accountOwner = "unknown"; GSAccessControlList acl = googleStorageService.getBucketAcl(bucketName); short bucketMode = GCSUtils.translateBucketAcl(acl, accountOwnerId);
@Override protected boolean deleteObject(String key) throws IOException { try { mClient.deleteObject(mBucketName, key); } catch (ServiceException e) { LOG.error("Failed to delete {}", key, e); return false; } return true; }
/** * Creates a new instance of {@link GCSInputStream}, at a specific position. * * @param bucketName the name of the bucket * @param key the key of the file * @param client the client for GCS * @param pos the position to start * @throws ServiceException if a service exception occurs */ GCSInputStream(String bucketName, String key, GoogleStorageService client, long pos) throws ServiceException { mBucketName = bucketName; mKey = key; mClient = client; mPos = pos; // For an empty file setting start pos = 0 will throw a ServiceException if (mPos > 0) { mObject = mClient.getObject(mBucketName, mKey, null, null, null, null, mPos, null); } else { mObject = mClient.getObject(mBucketName, mKey); } mInputStream = new BufferedInputStream(mObject.getDataInputStream()); }
GoogleStorageService gsService = new GoogleStorageService(gsCredentials); GSBucket[] myBuckets = gsService.listAllBuckets(); System.out.println("How many buckets to I have in GS? " + myBuckets.length); GSBucket testBucket = gsService.createBucket(BUCKET_NAME); System.out.println("Created test bucket: " + testBucket.getName()); object = gsService.putObject(BUCKET_NAME, object); gsService.putObject(BUCKET_NAME, stringObject); gsService.putObject(BUCKET_NAME, fileObject); gsService.putObject(BUCKET_NAME, helloWorldObject); GSObject objectDetailsOnly = gsService.getObjectDetails(BUCKET_NAME, TEST_OBJECT_NAME); System.out.println("GSObject, details only: " + objectDetailsOnly); GSObject objectComplete = gsService.getObject(BUCKET_NAME, TEST_OBJECT_NAME); System.out.println("GSObject, complete: " + objectComplete); GSObject downloadedObject = gsService.getObject(BUCKET_NAME, TEST_OBJECT_NAME); String textData = ServiceUtils.readInputStreamToString( downloadedObject.getDataInputStream(), "UTF-8"); GSBucket[] buckets = gsService.listAllBuckets(); GSObject[] objects = gsService.listObjects(buckets[b].getName()); GSObject[] filteredObjects = gsService.listObjects(BUCKET_NAME, prefix, delimiter);
public static void deleteBucket(String bucketName, String awsAccessKey, String awsSecretKey) { try { System.out.println(""); System.out.print("Deleting Bucket [" + bucketName + "]"); GSCredentials credentials = new GSCredentials(Main.cloudAccessKey, Main.cloudSecretKey); // To communicate with Google Storage use the GoogleStorageService. GoogleStorageService service = new GoogleStorageService(credentials); service.deleteBucket(bucketName); SDFSLogger.getLog().info("Bucket [" + bucketName + "] deleted"); System.out.println("Bucket [" + bucketName + "] deleted"); } catch (ServiceException e) { SDFSLogger.getLog() .warn("Unable to delete bucket " + bucketName, e); } }
@Override public void close() throws IOException { if (mClosed.getAndSet(true)) { return; } mLocalOutputStream.close(); try { GSObject obj = new GSObject(mKey); obj.setBucketName(mBucketName); obj.setDataInputFile(mFile); obj.setContentLength(mFile.length()); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); if (mHash != null) { obj.setMd5Hash(mHash.digest()); } else { LOG.warn("MD5 was not computed for: {}", mKey); } mClient.putObject(mBucketName, obj); if (!mFile.delete()) { LOG.error("Failed to delete temporary file @ {}", mFile.getPath()); } } catch (ServiceException e) { LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath()); throw new IOException(e); } } }
public void clearStore() throws IOException { try { SDFSLogger.getLog().warn( "Deleting all entries from Bucket [" + this.name + "]"); GSObject[] obj = gsService.listObjects(this.name); SDFSLogger.getLog().info("Will delete " + obj.length + " objects"); for (int i = 0; i < obj.length; i++) { gsService.deleteObject(this.name, obj[i].getKey()); } SDFSLogger.getLog().info( "All entries in bucket [" + this.getName() + "] deleted"); } catch (ServiceException e) { SDFSLogger.getLog().warn( "Unable to delete entries in " + this.getName(), e); throw new IOException(e); } }
/** * List all buckets in a given project * @param projectId The ID of the project being listed * @return a list of {@link GSBucket} * @throws ServiceException Service error */ public GSBucket[] listAllBuckets(String projectId) throws ServiceException { assertAuthenticatedConnection("List all buckets"); StorageBucket[] buckets = listAllBucketsImpl(projectId); MxDelegate.getInstance().registerStorageBucketMBeans(buckets); return GSBucket.cast(buckets); }
new CommandLineCredentialsProvider(), myProperties); } else if ("GS".equalsIgnoreCase(providerId)) { service = new GoogleStorageService( providerCredentials, APPLICATION_DESCRIPTION, new CommandLineCredentialsProvider(), myProperties);
@Override protected ObjectStatus getObjectStatus(String key) { try { GSObject meta = mClient.getObjectDetails(mBucketName, key); if (meta == null) { return null; } return new ObjectStatus(key, meta.getMd5HashAsBase64(), meta.getContentLength(), meta.getLastModifiedDate().getTime()); } catch (ServiceException e) { return null; } }
@Override protected boolean copyObject(String src, String dst) { LOG.debug("Copying {} to {}", src, dst); GSObject obj = new GSObject(dst); // Retry copy for a few times, in case some Jets3t or GCS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { mClient.copyObject(mBucketName, src, mBucketName, obj, false); return true; } catch (ServiceException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
GoogleStorageService gsService = new GoogleStorageService(gsCredentials); GSBucket[] myBuckets = gsService.listAllBuckets(); System.out.println("How many buckets to I have in GS? " + myBuckets.length); GSBucket testBucket = gsService.createBucket(BUCKET_NAME); System.out.println("Created test bucket: " + testBucket.getName()); object = gsService.putObject(BUCKET_NAME, object); gsService.putObject(BUCKET_NAME, stringObject); gsService.putObject(BUCKET_NAME, fileObject); gsService.putObject(BUCKET_NAME, helloWorldObject); GSObject objectDetailsOnly = gsService.getObjectDetails(BUCKET_NAME, TEST_OBJECT_NAME); System.out.println("GSObject, details only: " + objectDetailsOnly); GSObject objectComplete = gsService.getObject(BUCKET_NAME, TEST_OBJECT_NAME); System.out.println("GSObject, complete: " + objectComplete); GSObject downloadedObject = gsService.getObject(BUCKET_NAME, TEST_OBJECT_NAME); String textData = ServiceUtils.readInputStreamToString( downloadedObject.getDataInputStream(), "UTF-8"); GSBucket[] buckets = gsService.listAllBuckets(); GSObject[] objects = gsService.listObjects(buckets[b].getName()); GSObject[] filteredObjects = gsService.listObjects(BUCKET_NAME, prefix, delimiter);
@Override protected boolean createEmptyObject(String key) { try { GSObject obj = new GSObject(key); obj.setDataInputStream(new ByteArrayInputStream(new byte[0])); obj.setContentLength(0); obj.setMd5Hash(DIR_HASH); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); mClient.putObject(mBucketName, obj); return true; } catch (ServiceException e) { LOG.error("Failed to create directory: {}", key, e); return false; } }
/** * List all buckets in a given project * @param projectId The ID of the project being listed * @return a list of {@link GSBucket} * @throws ServiceException Service error */ public GSBucket[] listAllBuckets(String projectId) throws ServiceException { assertAuthenticatedConnection("List all buckets"); StorageBucket[] buckets = listAllBucketsImpl(projectId); MxDelegate.getInstance().registerStorageBucketMBeans(buckets); return GSBucket.cast(buckets); }
new CommandLineCredentialsProvider(), myProperties); } else if ("GS".equalsIgnoreCase(providerId)) { service = new GoogleStorageService( providerCredentials, APPLICATION_DESCRIPTION, new CommandLineCredentialsProvider(), myProperties);
@Override protected ObjectStatus getObjectStatus(String key) { try { GSObject meta = mClient.getObjectDetails(mBucketName, key); if (meta == null) { return null; } return new ObjectStatus(key, meta.getMd5HashAsBase64(), meta.getContentLength(), meta.getLastModifiedDate().getTime()); } catch (ServiceException e) { return null; } }
@Override protected boolean copyObject(String src, String dst) { LOG.debug("Copying {} to {}", src, dst); GSObject obj = new GSObject(dst); // Retry copy for a few times, in case some Jets3t or GCS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { mClient.copyObject(mBucketName, src, mBucketName, obj, false); return true; } catch (ServiceException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
GoogleStorageService googleStorageService = new GoogleStorageService(googleCredentials); StorageOwner storageOwner = googleStorageService.getAccountOwner(); String accountOwnerId = "unknown"; String accountOwner = "unknown"; GSAccessControlList acl = googleStorageService.getBucketAcl(bucketName); short bucketMode = GCSUtils.translateBucketAcl(acl, accountOwnerId);
@Override public void close() throws IOException { if (mClosed.getAndSet(true)) { return; } mLocalOutputStream.close(); try { GSObject obj = new GSObject(mKey); obj.setBucketName(mBucketName); obj.setDataInputFile(mFile); obj.setContentLength(mFile.length()); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); if (mHash != null) { obj.setMd5Hash(mHash.digest()); } else { LOG.warn("MD5 was not computed for: {}", mKey); } mClient.putObject(mBucketName, obj); if (!mFile.delete()) { LOG.error("Failed to delete temporary file @ {}", mFile.getPath()); } } catch (ServiceException e) { LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath()); throw new IOException(e); } } }
/** * This method leverages the ability to open a stream from GCS from a given offset. When the * underlying stream has fewer bytes buffered than the skip request, the stream is closed, and * a new stream is opened starting at the requested offset. * * @param n number of bytes to skip * @return the number of bytes skipped */ @Override public long skip(long n) throws IOException { if (mInputStream.available() >= n) { return mInputStream.skip(n); } // The number of bytes to skip is possibly large, open a new stream from GCS. mInputStream.close(); mPos += n; try { mObject = mClient.getObject(mBucketName, mKey, null /* ignore ModifiedSince */, null /* ignore UnmodifiedSince */, null /* ignore MatchTags */, null /* ignore NoneMatchTags */, mPos /* byteRangeStart */, null /* ignore byteRangeEnd */); mInputStream = new BufferedInputStream(mObject.getDataInputStream()); } catch (ServiceException e) { throw new IOException(e); } return n; } }