.payload( ephemeral ) .contentMD5(Files.hash( ephemeral, Hashing.md5() )) .contentType("application/json"); Blob blob = blobBuilder.build();
private static BlobMetadata createFakeBlobMetadata(BlobStore blobStore) { return blobStore.blobBuilder("fake-name") .build() .getMetadata(); }
BlobBuilder.PayloadBlobBuilder builder = blobStore .blobBuilder(blobName) .payload(payload); addContentMetdataFromHttpRequest(builder, request); builder.contentLength(payload.size()); throw new IllegalArgumentException(); } else { builder.tier(StorageClass.valueOf(storageClass).toTier()); builder.build().getMetadata(), options); blobStore.putBlob(containerName, builder.name(mpu.id()).build(), options); writeSimpleElement(xml, "UploadId", mpu.id());
public void publish(String remoteName, byte[] payload) throws IOException { Timer.Context ctx = uploadTimer.time(); try { Blob blob = blobStore.blobBuilder(remoteName).payload(payload) .contentType("application/json") .contentEncoding(remoteName.endsWith(".gz") ? "gzip" : "identity") .calculateMD5().build(); String containerName = CONTAINER_DATE_FORMAT.format(new Date()); if (!lastContainerCreated.matches(containerName)) { createContainer(); } blobStore.putBlob(containerName, blob); } finally { ctx.stop(); } } }
@Override public String putBlob(String containerName, Blob blob, PutOptions options) { long length; try (InputStream is = blob.getPayload().openStream()) { length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); } catch (IOException ioe) { throw new RuntimeException(ioe); } byte[] array = Longs.toByteArray(length); ByteSourcePayload payload = new ByteSourcePayload( ByteSource.wrap(array)); payload.setContentMetadata(blob.getPayload().getContentMetadata()); payload.getContentMetadata().setContentLength((long) array.length); payload.getContentMetadata().setContentMD5((HashCode) null); blob.setPayload(payload); return super.putBlob(containerName, blob, options); }
@Override public PageSet<? extends StorageMetadata> list(String container) { ImmutableSet.Builder<StorageMetadata> builder = ImmutableSet.builder(); PageSet<? extends StorageMetadata> pageSet = super.list(container); for (StorageMetadata sm : pageSet) { MutableStorageMetadata msm = new MutableStorageMetadataImpl(sm); msm.setSize(0L); builder.add(msm); } return new PageSetImpl<>(builder.build(), pageSet.getNextMarker()); }
@Override public String putBlob(final String containerName, Blob blob, final PutOptions options) { final String nearName = blob.getMetadata().getName(); String nearETag = writeStore.putBlob(containerName, blob, options); schedule(new Callable<String>() { @Override public String call() { Blob nearBlob = writeStore.getBlob(containerName, nearName); String farETag = delegate().putBlob(containerName, nearBlob, options); return farETag; } }); return nearETag; }
tempFile.deleteOnExit(); fop = new FileOutputStream(tempFile); InputStream is = blob.getPayload().openStream(); IOUtils.copyLarge(is, fop); return tempFile;
private NavigableMap<Long,String> getBlobsWithinRange(PageSet<? extends StorageMetadata> pages) { // TreeMap used because of sorted property TreeMap<Long, String> tsToBlobName = new TreeMap<Long, String>(); for (StorageMetadata blobMeta : pages) { String fileName = blobMeta.getName(); // 20140226_1393442533000.json.gz String dateAndTs = fileName.split("\\.", 2)[0].trim(); // 20140226_1393442533000 String tsCreated = dateAndTs.split("_")[1].trim(); // 1393442533000 long ts = Long.parseLong(tsCreated); tsToBlobName.put(ts, fileName); } //Gets key within the time range specified NavigableMap<Long, String> mapWithinRange = tsToBlobName.subMap(START_TIME - 60000*15, true, STOP_TIME + 60000*30, true); if(mapWithinRange.isEmpty()) { lastMarker = tsToBlobName.lastEntry().getValue().trim(); synchronized (CloudFilesManager.this) { // this is where we resume from. MarkerUtils.writeLastMarker(tsToBlobName.lastEntry().getValue().trim()); } } return mapWithinRange; }
final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list(); for ( Object o : blobStoreList.toArray() ) { StorageMetadata s = (StorageMetadata)o; if ( s.getName().startsWith( bucketPrefix )) { try { blobStore.deleteContainer(s.getName()); } catch ( ContainerNotFoundException cnfe ) { logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe ); logger.debug("Deleted bucket {}", s.getName());
@Override @Nullable public BlobMetadata blobMetadata(String container, String name) { Blob blob = getBlob(container, name); if (blob == null) { return null; } return blob.getMetadata(); }
@Override public List<MultipartPart> listMultipartUpload(MultipartUpload mpu) { ImmutableList.Builder<MultipartPart> builder = ImmutableList.builder(); for (MultipartPart part : super.listMultipartUpload(mpu)) { builder.add(MultipartPart.create(part.partNumber(), 0, part.partETag(), part.lastModified())); } return builder.build(); }
private static void handleAbortMultipartUpload(HttpServletResponse response, BlobStore blobStore, String containerName, String blobName, String uploadId) throws IOException, S3Exception { if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType( blobStore))) { if (!blobStore.blobExists(containerName, uploadId)) { throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD); } blobStore.removeBlob(containerName, uploadId); } // TODO: how to reconstruct original mpu? MultipartUpload mpu = MultipartUpload.create(containerName, blobName, uploadId, createFakeBlobMetadata(blobStore), new PutOptions()); blobStore.abortMultipartUpload(mpu); response.sendError(HttpServletResponse.SC_NO_CONTENT); }
public synchronized boolean hasNewFiles() { // see if there are any files since lastMarker. BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker); boolean emptiness = getBlobsWithinRange(pages).isEmpty(); if(emptiness) { log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME)); } else { log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME)); } return !emptiness; }
@Override public String completeMultipartUpload(final MultipartUpload mpu, final List<MultipartPart> parts) { long length = 0; for (MultipartPart part : parts) { length += part.partSize(); } byte[] array = Longs.toByteArray(length); ByteSourcePayload payload = new ByteSourcePayload( ByteSource.wrap(array)); super.abortMultipartUpload(mpu); MultipartPart part = delegate().uploadMultipartPart(mpu, 1, payload); return delegate().completeMultipartUpload(mpu, ImmutableList.of(part)); }
.payload( uploadFile ) .contentMD5(Files.hash( uploadFile, Hashing.md5())) .contentType( "application/json" ); Blob blob = blobBuilder.build();
@Override @Nullable public Blob getBlob(String container, String name, GetOptions options) { Blob blob = super.getBlob(container, name, options); if (blob == null) { return null; } byte[] array; try (InputStream is = blob.getPayload().openStream()) { array = ByteStreams.toByteArray(is); } catch (IOException ioe) { throw new RuntimeException(ioe); } long length = Longs.fromByteArray(array); ByteSourcePayload payload = new ByteSourcePayload( new NullByteSource().slice(0, length)); payload.setContentMetadata(blob.getPayload().getContentMetadata()); payload.getContentMetadata().setContentLength(length); payload.getContentMetadata().setContentMD5((HashCode) null); blob.setPayload(payload); return blob; }
@Override protected void after() { logger.debug("S3 proxy is stopping"); try { s3Proxy.stop(); BlobStore blobStore = blobStoreContext.getBlobStore(); for (StorageMetadata metadata : blobStore.list()) { blobStore.deleteContainer(metadata.getName()); } blobStoreContext.close(); } catch (Exception e) { throw new RuntimeException("Unable to stop S3 proxy", e); } FileUtils.deleteQuietly(blobStoreLocation); logger.debug("S3 proxy has stopped"); }
@Override public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { long length; try (InputStream is = payload.openStream()) { length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); } catch (IOException ioe) { throw new RuntimeException(ioe); } byte[] array = Longs.toByteArray(length); ByteSourcePayload newPayload = new ByteSourcePayload( ByteSource.wrap(array)); newPayload.setContentMetadata(payload.getContentMetadata()); newPayload.getContentMetadata().setContentLength((long) array.length); newPayload.getContentMetadata().setContentMD5((HashCode) null); MultipartPart part = super.uploadMultipartPart(mpu, partNumber, newPayload); return MultipartPart.create(part.partNumber(), length, part.partETag(), part.lastModified()); }