@Override protected InputStream open(long offset) { return getClient().getObject(new GetObjectRequest(_bk[0], _bk[1]).withRange(offset, _to)).getObjectContent(); }
BufferedImage image = ... ByteArrayOutputStream os = new ByteArrayOutputStream(); ImageIO.write(image, "png", os); byte[] buffer = os.toByteArray(); InputStream is = new ByteArrayInputStream(buffer); AmazonS3 s3 = new AmazonS3Client(new PropertiesCredentials(cred)); ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(buffer.length); s3.putObject(new PutObjectRequest("folder", key, is, meta));
AmazonS3 s3 = new AmazonS3Client(); // provide credentials, if need be ObjectListing objectListing = s3.listObjects(new ListObjectsRequest() .withBucketName("cdn.generalsentiment.com"); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey()); }
try { AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); String bucketName = getBucketName(); s3.createBucket(bucketName); S3Object object = s3.getObject(bucketName, getKey()); } catch (AmazonServiceException e) { String errorCode = e.getErrorCode(); if (!errorCode.equals("NoSuchKey")) { throw e; } Logger.getLogger(getClass()).debug("No such key!!!", e); }
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { initClient(); Collection<InetSocketAddress> addrs = new LinkedList<>(); try { ObjectListing list = keyPrefix == null ? s3.listObjects(bucketName) : s3.listObjects(bucketName, keyPrefix); while (true) { for (S3ObjectSummary sum : list.getObjectSummaries()) { InetSocketAddress addr = addr(sum); if (addr != null) addrs.add(addr); } if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break; } } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } return addrs; }
private Bucket createBucket(String bucket_name) { final AmazonS3 s3 = s3Client(); Bucket b = null; if (s3.doesBucketExistV2(bucket_name)) { System.out.format("Bucket %s already exists.\n", bucket_name); b = getBucket(bucket_name); } else { try { b = s3.createBucket(bucket_name); } catch (AmazonS3Exception e) { System.err.println(e.getErrorMessage()); } } return b; }
@Override public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception { S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) ); byte data[] = null; if ( offset == 0 && length == FIVE_MB ) { return object.getObjectContent(); } else { object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length)); } return new ByteArrayInputStream(data); }
/** {@inheritDoc} */ @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); initClient(); for (InetSocketAddress addr : addrs) { String key = key(addr); try { s3.putObject(bucketName, key, new ByteArrayInputStream(ENTRY_CONTENT), objMetadata); } catch (AmazonClientException e) { throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + ", entry=" + key + ']', e); } } }
ObjectMetadata om = new ObjectMetadata(); om.setContentLength(written); om.setContentType( mimeType ); PutObjectResult result = null; result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om ); InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest ); .withPartSize(partSize) .withLastPart(isLastPart); partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() ); partNumber++; MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest ); getS3Client().abortMultipartUpload( abortRequest ); Thread.sleep( 1000 ); timesIterated--; listResult = getS3Client().listMultipartUploads( listRequest ); if (logger.isDebugEnabled()) { logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString()); new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(), partETags ); CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request ); fileMetadata.put( AssetUtils.CONTENT_LENGTH, written ); fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() );
byte[] bI = org.apache.commons.codec.binary.Base64.decodeBase64((base64Data.substring(base64Data.indexOf(",")+1)).getBytes()); InputStream fis = new ByteArrayInputStream(bI); AmazonS3 s3 = new AmazonS3Client(); Region usWest02 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest02); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(bI.length); metadata.setContentType("image/png"); metadata.setCacheControl("public, max-age=31536000"); s3.putObject(BUCKET_NAME, filename, fis, metadata); s3.setObjectAcl(BUCKET_NAME, filename, CannedAccessControlList.PublicRead);
ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(bytes.length); objectMetadata.setContentMD5(new String(org.apache.commons.codec.binary.Base64.encodeBase64(DigestUtils.md5(bytes)))); retry.retry(() -> amazonS3.putObject( bucket, path, new ByteArrayInputStream(bytes), objectMetadata), MAX_RETRIES, RETRY_BACKOFF); retry.retry(() -> amazonS3.deleteObject(bucket, originalPath), MAX_RETRIES, RETRY_BACKOFF);
private void getFiles(String path) throws IOException{ AmazonS3 s3 = new AmazonS3Client( new BasicAWSCredentials(apiKey, apiKeySecret)); ObjectListing objList = s3.listObjects(bucket, path); for (S3ObjectSummary summary:objList.getObjectSummaries()) { //ignore folders if(! summary.getKey().endsWith(FOLDER_SUFFIX)){ S3Object obj = s3.getObject( new GetObjectRequest(bucket, summary.getKey())); logger.info("retrieving " + summary.getKey()); FileOutputStream fout = new FileOutputStream(TEMP_FOLDER + summary.getKey().substring(path.length())); InputStream in = obj.getObjectContent(); byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) > 0){ fout.write(buf, 0, len); } in.close(); fout.close(); } } } protected void afterPropertiesSet(GenericFileEndpoint<File> endpoint) throws Exception {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withDelimiter(DEFAULT_DELIMITER).withPrefix(prefix); listObjectsResponse = s3.listObjects(listObjectsRequest); } else { listObjectsResponse = s3.listNextBatchOfObjects(listObjectsResponse); for ( S3ObjectSummary s : listObjectsResponse.getObjectSummaries() ) { log.debug("Skipping " + s.getKey() + " as it does not match filter."); continue; if ( leavesRoot(destinationDirectory, s.getKey()) ) { throw new RuntimeException("Cannot download key " + s.getKey() + ", its relative path resolves outside the parent directory."); if ( !s.getKey().equals(prefix) && !listObjectsResponse.getCommonPrefixes().contains(s.getKey() + DEFAULT_DELIMITER) ) { objectSummaries.add(s); totalSize += s.getSize(); commonPrefixes.addAll(listObjectsResponse.getCommonPrefixes()); } while ( listObjectsResponse.isTruncated() ); } while ( !commonPrefixes.isEmpty() ); GetObjectRequest req = new GetObjectRequest(summary.getBucketName(), summary.getKey()) .<GetObjectRequest>withGeneralProgressListener( listener); downloads.add((DownloadImpl) doDownload(
@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix; s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred); s3.setEndpoint(bucketEndpoint); if (!s3.doesBucketExist(bucketName)) { try { s3.createBucket(bucketName); while (!s3.doesBucketExist(bucketName)) try { U.sleep(200); if (!s3.doesBucketExist(bucketName)) throw new IgniteSpiException("Failed to create bucket: " + bucketName, e); ObjectListing list = s3.listObjects(bucketName); for (S3ObjectSummary sum : list.getObjectSummaries()) { S3CheckpointData data = read(sum.getKey()); if (list.isTruncated()) list = s3.listNextBatchOfObjects(list); else break;
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
/** * Uploads the payload to the given s3 object key. * It is expected that the caller retrieves the object key using {@link #getLocation(Operation, PayloadType, String)} before making this call. * * @param path the s3 key of the object to be uploaded * @param payload an {@link InputStream} containing the json payload which is to be uploaded * @param payloadSize the size of the json payload in bytes */ @Override public void upload(String path, InputStream payload, long payloadSize) { try { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentType(CONTENT_TYPE); objectMetadata.setContentLength(payloadSize); PutObjectRequest request = new PutObjectRequest(bucketName, path, payload, objectMetadata); s3Client.putObject(request); } catch (SdkClientException e) { String msg = "Error communicating with S3"; logger.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } }
ObjectListing object_listing = s3.listObjects(bucket_name); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary)iterator.next(); s3.deleteObject(bucket_name, summary.getKey()); if (object_listing.isTruncated()) { object_listing = s3.listNextBatchOfObjects(object_listing); } else { break; VersionListing version_listing = s3.listVersions( new ListVersionsRequest().withBucketName(bucket_name)); while (true) { iterator.hasNext();) { S3VersionSummary vs = (S3VersionSummary)iterator.next(); s3.deleteVersion( bucket_name, vs.getKey(), vs.getVersionId()); version_listing = s3.listNextBatchOfVersions( version_listing); } else { s3.deleteBucket(bucket_name); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage());
final GetObjectRequest request; if (versionId == null) { request = new GetObjectRequest(bucket, key); } else { request = new GetObjectRequest(bucket, key, versionId); try (final S3Object s3Object = client.getObject(request)) { flowFile = session.importFrom(s3Object.getObjectContent(), flowFile); attributes.put("s3.bucket", s3Object.getBucketName()); final ObjectMetadata metadata = s3Object.getObjectMetadata(); if (metadata.getContentDisposition() != null) { final String fullyQualified = metadata.getContentDisposition(); final int lastSlash = fullyQualified.lastIndexOf("/"); if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) { attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1)); } else { attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
@Override public void addFile(String merchantStoreCode, InputContentFile inputStaticContentData) throws ServiceException { try { // get buckets String bucketName = bucketName(); String nodePath = nodePath(merchantStoreCode, inputStaticContentData.getFileContentType()); final AmazonS3 s3 = s3Client(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentType(inputStaticContentData.getMimeType()); PutObjectRequest request = new PutObjectRequest(bucketName, nodePath + inputStaticContentData.getFileName(), inputStaticContentData.getFile(), metadata); request.setCannedAcl(CannedAccessControlList.PublicRead); s3.putObject(request); LOGGER.info("Content add file"); } catch (final Exception e) { LOGGER.error("Error while adding file", e); throw new ServiceException(e); } }