private ObjectMetadata downloadTo0(final OutputStream output, RequestMetricCollector requestMetricCollector) { GetObjectRequest req = new GetObjectRequest(getBucketName(), getKey()) .withRequestMetricCollector(requestMetricCollector); S3Object s3Object = getAmazonS3Client().getObject(req); S3ObjectInputStream objectContent = s3Object.getObjectContent(); try { byte[] buffer = new byte[1024 * 10]; int bytesRead = -1; while ((bytesRead = objectContent.read(buffer)) > -1) { output.write(buffer, 0, bytesRead); } } catch (IOException ioe) { objectContent.abort(); throw new SdkClientException("Unable to transfer content from Amazon S3 to the output stream", ioe); } finally { try { objectContent.close(); } catch (IOException ioe) {} } return s3Object.getObjectMetadata(); }
throw new SdkClientException("testing"); throw ace; } else { LOG.info("Retry the download of object " + s3Object.getKey() + " (bucket " + s3Object.getBucketName() + ")", ace); hasRetried = true; s3Object.getObjectContent().abort();
/** * Download an object from S3. * * @param bucket * The name of the bucket * @param key * The file key of the object to upload/update. * @param result * The Hash map where data from the object are written * */ protected Status readFromStorage(String bucket, String key, Map<String, ByteIterator> result, SSECustomerKey ssecLocal) { try { S3Object object = getS3ObjectAndMetadata(bucket, key, ssecLocal); InputStream objectData = object.getObjectContent(); //consuming the stream // writing the stream to bytes and to results result.put(key, new ByteArrayByteIterator(IOUtils.toByteArray(objectData))); objectData.close(); object.close(); } catch (Exception e){ System.err.println("Not possible to get the object "+key); e.printStackTrace(); return Status.ERROR; } return Status.OK; }
POINT_DIR.mkdirs(); S3Object obj = s3.getObject(pointsetBucket, pointSetId + ".json.gz"); ObjectMetadata objMet = obj.getObjectMetadata(); FileOutputStream fos = new FileOutputStream(cachedFile); GZIPInputStream gis = new GZIPInputStream(obj.getObjectContent()); try { ByteStreams.copy(gis, fos);
private static boolean loadConfigS3(Configuration conf, String confFilePath) throws IOException { Matcher matcher = s3pattern.matcher(confFilePath); matcher.matches(); String bucket = matcher.group(1); String key = matcher.group(2); AmazonS3 s3 = new AmazonS3Client(new PropertiesCredentials(Utility.class.getResourceAsStream("AwsCredentials.properties"))); S3Object object = s3.getObject(new GetObjectRequest(bucket, key)); InputStream is = object.getObjectContent(); Properties configProps = new Properties(); configProps.load(is); for (Object keyObj : configProps.keySet()){ String keySt = keyObj.toString(); conf.set(keySt, configProps.getProperty(keySt)); } return true; }
@Override protected InputStream open(long offset) { return getClient().getObject(new GetObjectRequest(_bk[0], _bk[1]).withRange(offset, _to)).getObjectContent(); }
/** * Downloads the payload stored in the s3 object. * * @param path the S3 key of the object * @return an input stream containing the contents of the object * Caller is expected to close the input stream. */ @Override public InputStream download(String path) { try { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, path)); return s3Object.getObjectContent(); } catch (SdkClientException e) { String msg = "Error communicating with S3"; logger.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } }
final GetObjectRequest request; if (versionId == null) { request = new GetObjectRequest(bucket, key); } else { request = new GetObjectRequest(bucket, key, versionId); try (final S3Object s3Object = client.getObject(request)) { flowFile = session.importFrom(s3Object.getObjectContent(), flowFile); attributes.put("s3.bucket", s3Object.getBucketName()); final ObjectMetadata metadata = s3Object.getObjectMetadata(); if (metadata.getContentDisposition() != null) { final String fullyQualified = metadata.getContentDisposition(); final int lastSlash = fullyQualified.lastIndexOf("/"); if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) { attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1)); } else { attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition());
@Override public InputStream call() throws IOException { final S3Object s3 = getS3Client().getObject(new GetObjectRequest(bucket, key)); return s3.getObjectContent(); } });
public Integer retriableCall() throws IOException { GetObjectRequest req = new GetObjectRequest(bucketName, remotePath); req.setRange(firstByte, endByte); try (S3ObjectInputStream is = s3Client.getObject(req).getObjectContent()) { byte[] readBuf = new byte[4092]; int rCnt; int readTotal = 0; int incomingOffet = off; while ((rCnt = is.read(readBuf, 0, readBuf.length)) >= 0) { System.arraycopy(readBuf, 0, b, incomingOffet, rCnt); readTotal += rCnt; incomingOffet += rCnt; } if (readTotal == 0 && rCnt == -1) return -1; offset += readTotal; return readTotal; } } }.call();
byte[] buffer = new byte[1024*10]; int bytesRead; while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) { outputStream.write(buffer, 0, bytesRead); s3Object.getObjectContent().abort(); throw new SdkClientException( "Unable to store object contents to disk: " + e.getMessage(), e); } finally { closeQuietly(outputStream, LOG); FileLocks.unlock(dstfile); closeQuietly(s3Object.getObjectContent(), LOG); byte[] serverSideHash = null; try { final ObjectMetadata metadata = s3Object.getObjectMetadata(); if (!skipMd5CheckStrategy.skipClientSideValidationPerGetResponse(metadata)) { clientSideHash = Md5Utils.computeMD5Hash(new FileInputStream(dstfile)); serverSideHash = BinaryUtils.fromHex(metadata.getETag()); throw new SdkClientException("Unable to verify integrity of data download. " + "Client calculated content hash didn't match hash calculated by Amazon S3. " + "The data stored in '" + dstfile.getAbsolutePath() + "' may be corrupt.");
@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
@Override public S3Object getObject(GetObjectRequest getObjectRequest) throws SdkClientException, AmazonServiceException { getObjectRequest = beforeClientExecution(getObjectRequest); assertNotNull(getObjectRequest, "GetObjectRequest"); assertStringNotEmpty(getObjectRequest.getBucketName(), "BucketName"); assertStringNotEmpty(getObjectRequest.getKey(), "Key"); Request<GetObjectRequest> request = createRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest, HttpMethodName.GET); request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "GetObject"); addPartNumberIfNotNull(request, getObjectRequest.getPartNumber()); s3Object.setBucketName(getObjectRequest.getBucketName()); s3Object.setKey(getObjectRequest.getKey()); InputStream is = s3Object.getObjectContent(); HttpRequestBase httpRequest = s3Object.getObjectContent().getHttpRequest(); if (!skipMd5CheckStrategy.skipClientSideValidation(getObjectRequest, s3Object.getObjectMetadata())) { byte[] serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag()); try { s3Object.getObjectMetadata().getContentLength(), // expected length new S3AbortableInputStream(is, httpRequest, s3Object.getObjectMetadata().getContentLength()); s3Object.setObjectContent(new S3ObjectInputStream(abortableInputStream, httpRequest, false)); return s3Object; } catch (AmazonS3Exception ase) {
@Override public InputStream getInputStream() throws IOException { GetObjectRequest getObjectRequest = new GetObjectRequest(this.bucketName, this.objectName); if (this.versionId != null) { getObjectRequest.setVersionId(this.versionId); } return this.amazonS3.getObject(getObjectRequest).getObjectContent(); }
@Override public ObjectMetadata getObjectSecurely(GetObjectRequest getObjectRequest, File destinationFile) { assertParameterNotNull(destinationFile, "The destination file parameter must be specified when downloading an object directly to a file"); S3Object s3Object = getObjectSecurely(getObjectRequest); // getObject can return null if constraints were specified but not met if (s3Object == null) return null; OutputStream outputStream = null; try { outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile)); byte[] buffer = new byte[1024*10]; int bytesRead; while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) { outputStream.write(buffer, 0, bytesRead); } } catch (IOException e) { throw new SdkClientException( "Unable to store object contents to disk: " + e.getMessage(), e); } finally { closeQuietly(outputStream, log); closeQuietly(s3Object.getObjectContent(), log); } /* * Unlike the standard Amazon S3 Client, the Amazon S3 Encryption Client does not do an MD5 check * here because the contents stored in S3 and the contents we just retrieved are different. In * S3, the stored contents are encrypted, and locally, the retrieved contents are decrypted. */ return s3Object.getObjectMetadata(); }
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
@Override public String getObjectAsString(String bucketName, String key) throws AmazonServiceException, SdkClientException { rejectNull(bucketName, "Bucket name must be provided"); rejectNull(key, "Object key must be provided"); S3Object object = getObject(bucketName, key); try { return IOUtils.toString(object.getObjectContent()); } catch (IOException e) { throw new SdkClientException("Error streaming content from S3 during download"); } finally { IOUtils.closeQuietly(object, log); } }
try { S3Object object = getS3ObjectAndMetadata(bucket, key, ssecLocal); int sizeOfFile = (int)object.getObjectMetadata().getContentLength(); fieldCount = sizeOfFile/sizeArray; totalSize = sizeOfFile; object.close(); } catch (Exception e){ System.err.println("Not possible to get the object :"+key); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(totalSize); PutObjectRequest putObjectRequest = null; if (sseLocal.equals("true")) { s3Client.putObject(putObjectRequest); if(res.getETag() == null) { return Status.ERROR;
private synchronized void reopen(long pos) throws IOException { if (wrappedStream != null) { if (LOG.isDebugEnabled()) { LOG.debug("Aborting old stream to open at pos " + pos); } wrappedStream.abort(); } LOG.info("Actually opening file " + key + " at pos " + pos); GetObjectRequest request = new GetObjectRequest(bucket, key); request.setRange(pos, contentLength-1); wrappedObject = client.getObject(request); wrappedStream = wrappedObject.getObjectContent(); if (wrappedStream == null) { throw new IOException("Null IO stream"); } this.pos = pos; }