S3ObjectInputStream getObjectContent() { return s3obj.getObjectContent(); }
/** * Releases any underlying system resources. If the resources * are already released then invoking this method has no effect. * * @throws IOException if an I/O error occurs */ @Override public void close() throws IOException { InputStream is = getObjectContent(); if (is != null) is.close(); }
/** * Converts and return the underlying S3 object as a json string. * * @throws SdkClientException if failed in JSON conversion. */ String toJsonString() { try { return from(s3obj.getObjectContent()); } catch (Exception e) { throw new SdkClientException("Error parsing JSON: " + e.getMessage()); } }
private void downloadGraphSourceFiles(String graphId, File dir) throws IOException { File graphCacheDir = dir; if (!graphCacheDir.exists()) graphCacheDir.mkdirs(); File graphZipFile = new File(graphCacheDir, graphId + ".zip"); File extractedGraphDir = new File(graphCacheDir, graphId); if (extractedGraphDir.exists()) { FileUtils.deleteDirectory(extractedGraphDir); } extractedGraphDir.mkdirs(); S3Object graphZip = s3.getObject(graphBucket, graphId+".zip"); InputStream zipFileIn = graphZip.getObjectContent(); OutputStream zipFileOut = new FileOutputStream(graphZipFile); IOUtils.copy(zipFileIn, zipFileOut); IOUtils.closeQuietly(zipFileIn); IOUtils.closeQuietly(zipFileOut); unpackGraphZip(graphZipFile, extractedGraphDir); }
@Override protected InputStream openObjectStream(URI object) throws IOException { try { // Get data of the given object and open an input stream final String bucket = object.getAuthority(); final String key = S3Utils.extractS3Key(object); final S3Object s3Object = s3Client.getObject(bucket, key); if (s3Object == null) { throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", bucket, key); } return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); } }
@Override public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception { S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) ); byte data[] = null; if ( offset == 0 && length == FIVE_MB ) { return object.getObjectContent(); } else { object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length)); } return new ByteArrayInputStream(data); }
/** * Parses instruction data retrieved from S3 and returns a JSON string * representing the instruction. Made for testing purposes. */ static String parseInstructionFile(S3Object instructionFile) { try { return convertStreamToString(instructionFile.getObjectContent()); } catch (Exception e) { throw failure(e, "Error parsing JSON instruction file"); } }
/** * Cancels this download. * * @throws IOException */ public synchronized void abort() throws IOException { this.monitor.getFuture().cancel(true); if ( s3Object != null ) { s3Object.getObjectContent().abort(); } setState(TransferState.Canceled); }
@Override protected InputStream openObjectStream(URI object, long start) throws IOException { final String bucket = object.getAuthority(); final String key = S3Utils.extractS3Key(object); final GetObjectRequest request = new GetObjectRequest(bucket, key); request.setRange(start); try { final S3Object s3Object = s3Client.getObject(request); if (s3Object == null) { throw new ISE( "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", bucket, key, start ); } return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); } }
/** * Downloads the payload stored in the s3 object. * * @param path the S3 key of the object * @return an input stream containing the contents of the object * Caller is expected to close the input stream. */ @Override public InputStream download(String path) { try { S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, path)); return s3Object.getObjectContent(); } catch (SdkClientException e) { String msg = "Error communicating with S3"; logger.error(msg, e); throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e); } }
@Override public String getObjectAsString(String bucketName, String key) throws AmazonServiceException, SdkClientException { rejectNull(bucketName, "Bucket name must be provided"); rejectNull(key, "Object key must be provided"); S3Object object = getObject(bucketName, key); try { return IOUtils.toString(object.getObjectContent()); } catch (IOException e) { throw new SdkClientException("Error streaming content from S3 during download"); } finally { IOUtils.closeQuietly(object, log); } }
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */ public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest( s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
@Override protected InputStream open(long offset) { return getClient().getObject(new GetObjectRequest(_bk[0], _bk[1]).withRange(offset, _to)).getObjectContent(); }
try { GetObjectRequest request = new GetObjectRequest(host, keyFromPath(path)).withRange(start); return s3.getObject(request).getObjectContent();
/** * Opens a new stream at mPos if the wrapped stream mIn is null. */ private void openStream() { if (mIn != null) { // stream is already open return; } GetObjectRequest getReq = new GetObjectRequest(mBucketName, mKey); // If the position is 0, setting range is redundant and causes an error if the file is 0 length if (mPos > 0) { getReq.setRange(mPos); } AmazonS3Exception lastException = null; while (mRetryPolicy.attempt()) { try { mIn = mClient.getObject(getReq).getObjectContent(); return; } catch (AmazonS3Exception e) { LOG.warn("Attempt {} to open key {} in bucket {} failed with exception : {}", mRetryPolicy.getAttemptCount(), mKey, mBucketName, e.toString()); if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) { throw e; } // Key does not exist lastException = e; } } // Failed after retrying key does not exist throw lastException; }
private ObjectMetadata downloadTo0(final OutputStream output, RequestMetricCollector requestMetricCollector) { GetObjectRequest req = new GetObjectRequest(getBucketName(), getKey()) .withRequestMetricCollector(requestMetricCollector); S3Object s3Object = getAmazonS3Client().getObject(req); S3ObjectInputStream objectContent = s3Object.getObjectContent(); try { byte[] buffer = new byte[1024 * 10]; int bytesRead = -1; while ((bytesRead = objectContent.read(buffer)) > -1) { output.write(buffer, 0, bytesRead); } } catch (IOException ioe) { objectContent.abort(); throw new SdkClientException("Unable to transfer content from Amazon S3 to the output stream", ioe); } finally { try { objectContent.close(); } catch (IOException ioe) {} } return s3Object.getObjectMetadata(); }
@Override public InputStream openStream() throws IOException { try { final long start; final long end = objectMetadata.getContentLength() - 1; if (offset > 0 && offset < objectMetadata.getContentLength()) { start = offset; } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) { start = objectMetadata.getContentLength() + offset; } else { start = 0; } final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey) .withMatchingETagConstraint(objectMetadata.getETag()) .withRange(start, end); return service.getObject(request).getObjectContent(); } catch (AmazonServiceException e) { throw new IOException(e); } } }
/** * Returns an input stream for a s3 object. The returned input stream is not thread-safe. */ @Override public InputStream openInputStream() throws IOException { try { if (s3Object == null) { // lazily promote to full GET s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); } final InputStream in = s3Object.getObjectContent(); final Closer closer = Closer.create(); closer.register(in); closer.register(s3Object); return new FilterInputStream(in) { @Override public void close() throws IOException { closer.close(); } }; } catch (AmazonServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } }
@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
@Override public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest selectRequest) throws AmazonServiceException, SdkClientException { selectRequest = beforeClientExecution(selectRequest); rejectNull(selectRequest, "The request parameter must be specified"); rejectNull(selectRequest.getBucketName(), "The bucket name parameter must be specified when selecting object content."); rejectNull(selectRequest.getKey(), "The key parameter must be specified when selecting object content."); Request<SelectObjectContentRequest> request = createRequest(selectRequest.getBucketName(), selectRequest.getKey(), selectRequest, HttpMethodName.POST); request.addParameter("select", null); request.addParameter("select-type", "2"); populateSSE_C(request, selectRequest.getSSECustomerKey()); setContent(request, RequestXmlFactory.convertToXmlByteArray(selectRequest), ContentType.APPLICATION_XML.toString(), true); S3Object result = invoke(request, new S3ObjectResponseHandler(), selectRequest.getBucketName(), selectRequest.getKey()); // Hold a reference to this client while the InputStream is still // around - otherwise a finalizer in the HttpClient may reset the // underlying TCP connection out from under us. SdkFilterInputStream resultStream = new ServiceClientHolderInputStream(result.getObjectContent(), this); return new SelectObjectContentResult().withPayload(new SelectObjectContentEventStream(resultStream)); }