while (offset < objectSize) { if (offset + length > objectSize) length = objectSize - offset; futures.add(executorService.submit(new DownloadPartTask(Range.fromOffsetLength(offset, length), channel))); offset += length;
@Override public InputStream readObjectStream(String bucketName, String key, Range range) { byte[] bytes = new byte[Math.toIntExact(range.getLast() + 1 - range.getFirst())]; Path path = Paths.get(this.baseDir, bucketName, key); FileInputStream returnStream; try { returnStream = new FileInputStream(path.toFile()); if (range.getFirst() != 0) { long bytesSkipped = 0; do { bytesSkipped += returnStream.skip(range.getFirst()); } while (bytesSkipped < range.getFirst()); } StreamHelpers.readAll(returnStream, bytes, 0, bytes.length); return new ByteArrayInputStream(bytes); } catch (IOException e) { throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", ""); } }
public static Range fromOffsetLength(long offset, long length) { return new Range(offset, offset + length - 1); }
@Override public CopyPartResult copyPart(CopyPartRequest request) { Range range = request.getSourceRange(); if (range.getLast() == -1) { range = Range.fromOffsetLength(0, aclMap.get(request.getSourceKey()).getSize()); request.withSourceRange(range); } CopyObjectResult result = executeRequest(client, request, CopyObjectResult.class); CopyPartResult retVal = new CopyPartResult(); retVal.setPartNumber(request.getPartNumber()); retVal.setETag(result.getETag()); return retVal; } }
@Override public Void call() throws Exception { InputStream is = s3Client.readObjectStream(bucket, key, range); try { is = new ProgressInputStream(is, LargeFileDownloader.this); byte[] buffer = new byte[32 * 1024]; long pos = range.getFirst(); for (int r = 0; r != -1; r = is.read(buffer)) { channel.write(ByteBuffer.wrap(buffer, 0, r), pos); pos += r; } return null; } finally { try { is.close(); } catch (Throwable t) { log.warn("could not close object stream", t); } } } }
@Override public Map<String, List<Object>> getHeaders() { Map<String, List<Object>> headers = super.getHeaders(); if (range != null) RestUtil.putSingle(headers, RestUtil.HEADER_RANGE, "bytes=" + range.toString()); if (objectMetadata != null) headers.putAll(objectMetadata.toHeaders()); if (ifModifiedSince != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_MODIFIED_SINCE, RestUtil.headerFormat(ifModifiedSince)); if (ifUnmodifiedSince != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_UNMODIFIED_SINCE, RestUtil.headerFormat(ifUnmodifiedSince)); if (ifMatch != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_MATCH, ifMatch); if (ifNoneMatch != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_NONE_MATCH, ifNoneMatch); if (acl != null) headers.putAll(acl.toHeaders()); if (cannedAcl != null) RestUtil.putSingle(headers, S3Constants.AMZ_ACL, cannedAcl.getHeaderValue()); return headers; }
@Override public PutObjectResult putObject(PutObjectRequest request) { if (request.getObjectMetadata() != null) { request.setObjectMetadata(null); } try { Path path = Paths.get(this.baseDir, request.getBucketName(), request.getKey()); Path parent = path.getParent(); assert parent != null; Files.createDirectories(parent); Files.createFile(path); } catch (IOException e) { throw new S3Exception(e.getMessage(), 0, e); } PutObjectResult retVal = new PutObjectResult(); if (request.getAcl() != null) { long size = 0; if (request.getRange() != null) { size = request.getRange().getLast() + 1; } aclMap.putIfAbsent(request.getKey(), new AclSize(request.getAcl(), size)); } return retVal; }
@Override public long appendObject(String bucketName, String key, Object content) { return putObject(new PutObjectRequest(bucketName, key, content) .withRange(Range.fromOffset(-1))).getAppendOffset(); }
@Override public Map<String, List<Object>> getHeaders() { Map<String, List<Object>> headers = super.getHeaders(); if (range != null) RestUtil.putSingle(headers, RestUtil.HEADER_RANGE, "bytes=" + range.toString()); if (ifModifiedSince != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_MODIFIED_SINCE, RestUtil.headerFormat(ifModifiedSince)); if (ifUnmodifiedSince != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_UNMODIFIED_SINCE, RestUtil.headerFormat(ifUnmodifiedSince)); if (ifMatch != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_MATCH, ifMatch); if (ifNoneMatch != null) RestUtil.putSingle(headers, RestUtil.HEADER_IF_NONE_MATCH, ifNoneMatch); return headers; }
@Override public PutObjectResult putObject(PutObjectRequest request) { S3ObjectMetadata metadata = request.getObjectMetadata(); if (request.getObjectMetadata() != null) { request.setObjectMetadata(null); } PutObjectResult retVal = client.putObject(request); if (request.getAcl() != null) { long size = 0; if (request.getRange() != null) { size = request.getRange().getLast() - 1; } aclMap.put(request.getKey(), new AclSize(request.getAcl(), size)); } return retVal; }
@Override public String call() throws Exception { Range range = Range.fromOffsetLength(offset, length); InputStream is = file != null ? new FileInputStream(file) : stream; is = new ProgressInputStream(is, LargeFileUploader.this); SizedInputStream segmentStream = file != null ? new InputStreamSegment(is, offset, length) : new SizedInputStream(is, length); PutObjectRequest request = new PutObjectRequest(bucket, key, segmentStream).withRange(range); return s3Client.putObject(request).getETag(); } }
@Synchronized @Override public void putObject(String bucketName, String key, Range range, Object content) { Path path = Paths.get(this.baseDir, bucketName, key); try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) { long startOffset = range.getFirst(); long length = range.getLast() + 1 - range.getFirst(); do { long bytesTransferred = channel.transferFrom(Channels.newChannel((InputStream) content), range.getFirst(), range.getLast() + 1 - range.getFirst()); length -= bytesTransferred; startOffset += bytesTransferred; } while (length > 0); AclSize aclKey = aclMap.get(key); aclMap.put(key, aclKey.withSize(range.getLast() + 1)); } catch (IOException e) { throw new S3Exception("NoObject", 404, "NoSuchKey", key); } }
public static Range fromOffset(long offset) { return new Range(offset, null); }
private int doRead(SegmentHandle handle, long offset, byte[] buffer, int bufferOffset, int length) throws Exception { long traceId = LoggerHelpers.traceEnter(log, "read", handle.getSegmentName(), offset, bufferOffset, length); if (offset < 0 || bufferOffset < 0 || length < 0) { throw new ArrayIndexOutOfBoundsException(); } try (InputStream reader = client.readObjectStream(config.getBucket(), config.getRoot() + handle.getSegmentName(), Range.fromOffsetLength(offset, length))) { /* * TODO: This implementation assumes that if S3Client.readObjectStream returns null, then * the object does not exist and we throw StreamNotExistsException. The javadoc, however, * says that this call returns null in case of 304 and 412 responses. We need to * investigate what these responses mean precisely and react accordingly. * * See https://github.com/pravega/pravega/issues/1549 */ if (reader == null) { throw new StreamSegmentNotExistsException(handle.getSegmentName()); } int bytesRead = StreamHelpers.readAll(reader, buffer, bufferOffset, length); LoggerHelpers.traceLeave(log, "read", traceId, bytesRead); return bytesRead; } }
@Synchronized @Override public void putObject(String bucketName, String key, Range range, Object content) { byte[] totalByes = new byte[Math.toIntExact(range.getLast() + 1)]; try { if (range.getFirst() != 0) { int bytesRead = client.getObject(bucketName, key).getObject().read(totalByes, 0, Math.toIntExact(range.getFirst())); if (bytesRead != range.getFirst()) { throw new IllegalStateException("Unable to read from the object " + key); } } int bytesRead = ((InputStream) content).read(totalByes, Math.toIntExact(range.getFirst()), Math.toIntExact(range.getLast() + 1 - range.getFirst())); if (bytesRead != range.getLast() + 1 - range.getFirst()) { throw new IllegalStateException("Not able to read from input stream."); } client.putObject(new PutObjectRequest(bucketName, key, (Object) new ByteArrayInputStream(totalByes))); aclMap.put(key, aclMap.get(key).withSize(range.getLast() - 1)); } catch (IOException e) { throw new S3Exception("NoObject", HttpStatus.SC_NOT_FOUND, "NoSuchKey", key); } }
private Void doWrite(SegmentHandle handle, long offset, InputStream data, int length) throws StreamSegmentException { Preconditions.checkArgument(!handle.isReadOnly(), "handle must not be read-only."); long traceId = LoggerHelpers.traceEnter(log, "write", handle.getSegmentName(), offset, length); SegmentProperties si = doGetStreamSegmentInfo(handle.getSegmentName()); if (si.isSealed()) { throw new StreamSegmentSealedException(handle.getSegmentName()); } if (si.getLength() != offset) { throw new BadOffsetException(handle.getSegmentName(), si.getLength(), offset); } client.putObject(this.config.getBucket(), this.config.getRoot() + handle.getSegmentName(), Range.fromOffsetLength(offset, length), data); LoggerHelpers.traceLeave(log, "write", traceId); return null; }
@Synchronized @Override public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request) { Map<Integer, CopyPartRequest> partMap = multipartUploads.get(request.getKey()); if (partMap == null) { throw new S3Exception("NoSuchKey", HttpStatus.SC_NOT_FOUND, "NoSuchKey", ""); } try { partMap.forEach((index, copyPart) -> { if (!copyPart.getKey().equals(copyPart.getSourceKey())) { Path sourcePath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getSourceKey()); Path targetPath = Paths.get(this.baseDir, copyPart.getBucketName(), copyPart.getKey()); try (FileChannel sourceChannel = FileChannel.open(sourcePath, StandardOpenOption.READ); FileChannel targetChannel = FileChannel.open(targetPath, StandardOpenOption.WRITE)) { targetChannel.transferFrom(sourceChannel, Files.size(targetPath), copyPart.getSourceRange().getLast() + 1 - copyPart.getSourceRange().getFirst()); targetChannel.close(); AclSize aclMap = this.aclMap.get(copyPart.getKey()); this.aclMap.put(copyPart.getKey(), aclMap.withSize(Files.size(targetPath))); } catch (IOException e) { throw new S3Exception("NoSuchKey", 404, "NoSuchKey", ""); } } }); } finally { multipartUploads.remove(request.getKey()); } return new CompleteMultipartUploadResult(); }
targetPath, uploadId, 1).withSourceRange(Range.fromOffsetLength(0, offset)); CopyPartResult copyResult = client.copyPart(copyRequest); targetPath, uploadId, 2).withSourceRange(Range.fromOffsetLength(0, objectSize));