@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
long readSnapshotIndexLatestBlob() throws IOException { try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); return Numbers.bytesToLong(out.bytes().toBytesRef()); } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(inputStream, out); final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
try (InputStream blob = blobContainer().readBlob(snapshotsIndexBlobName)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); try (InputStream blob = blobContainer().readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out);
@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
public static byte[] readBlobFully(BlobContainer container, String name, int length) throws IOException { byte[] data = new byte[length]; try (InputStream inputStream = container.readBlob(name)) { assertThat(inputStream.read(data), equalTo(length)); assertThat(inputStream.read(), equalTo(-1)); } return data; }
/** * Reads and parses the blob with given name. * * If required the checksum of the blob will be verified. * * @param blobContainer blob container * @param blobName blob name * @return parsed blob object */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { return read(new BytesArray(ByteStreams.toByteArray(inputStream))); } } }
/** * Reads and parses the blob with given name. * * If required the checksum of the blob will be verified. * * @param blobContainer blob container * @param blobName blob name * @return parsed blob object */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(inputStream, out); return read(out.bytes()); } } }
long readSnapshotIndexLatestBlob() throws IOException { try (InputStream blob = snapshotsBlobContainer.readBlob(INDEX_LATEST_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); return Numbers.bytesToLong(out.bytes().toBytesRef()); } }
long readSnapshotIndexLatestBlob() throws IOException { try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); return Numbers.bytesToLong(out.bytes().toBytesRef()); } }
long readSnapshotIndexLatestBlob() throws IOException { try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); return Numbers.bytesToLong(out.bytes().toBytesRef()); } }
/** * Reads snapshot index file * <p> * This file can be used by read-only repositories that are unable to list files in the repository * * @return list of snapshots in the repository * @throws IOException I/O errors */ protected List<SnapshotId> readSnapshotList() throws IOException { try (InputStream blob = snapshotsBlobContainer.readBlob(SNAPSHOTS_FILE)) { final byte[] data = ByteStreams.toByteArray(blob); ArrayList<SnapshotId> snapshots = new ArrayList<>(); try (XContentParser parser = XContentHelper.createParser(new BytesArray(data))) { if (parser.nextToken() == XContentParser.Token.START_OBJECT) { if (parser.nextToken() == XContentParser.Token.FIELD_NAME) { String currentFieldName = parser.currentName(); if ("snapshots".equals(currentFieldName)) { if (parser.nextToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { snapshots.add(new SnapshotId(repositoryName, parser.text())); } } } } } } return Collections.unmodifiableList(snapshots); } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(inputStream, out); final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(inputStream, out); final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(inputStream, out); final byte[] bytes = out.toByteArray(); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }
public void testWriteRead() throws IOException { try(BlobStore store = newBlobStore()) { final BlobContainer container = store.blobContainer(new BlobPath()); byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); writeBlob(container, "foobar", new BytesArray(data)); try (InputStream stream = container.readBlob("foobar")) { BytesRefBuilder target = new BytesRefBuilder(); while (target.length() < data.length) { byte[] buffer = new byte[scaledRandomIntBetween(1, data.length - target.length())]; int offset = scaledRandomIntBetween(0, buffer.length - 1); int read = stream.read(buffer, offset, buffer.length - offset); target.append(new BytesRef(buffer, offset, read)); } assertEquals(data.length, target.length()); assertArrayEquals(data, Arrays.copyOfRange(target.bytes(), 0, target.length())); } } }
/** * Reads blob with specified name without resolving the blobName using using {@link #blobName} method. * * @param blobContainer blob container * @param blobName blob name */ public T readBlob(BlobContainer blobContainer, String blobName) throws IOException { try (InputStream inputStream = blobContainer.readBlob(blobName)) { byte[] bytes = ByteStreams.toByteArray(inputStream); final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")"; try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) { CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION); long filePointer = indexInput.getFilePointer(); long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer; BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize); return read(bytesReference); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // we trick this into a dedicated exception with the original stacktrace throw new CorruptStateException(ex); } } }