/** * Reads blob content from the input stream and writes it to the container in a new blob with the given name, * using an atomic write operation if the implementation supports it. When the BlobContainer implementation * does not provide a specific implementation of writeBlobAtomic(String, InputStream, long), then * the {@link #writeBlob(String, InputStream, long, boolean)} method is used. * * This method assumes the container does not already contain a blob of the same blobName. If a blob by the * same name already exists, the operation will fail and an {@link IOException} will be thrown. * * @param blobName * The name of the blob to write the contents of the input stream to. * @param inputStream * The input stream from which to retrieve the bytes to write to the blob. * @param blobSize * The size of the blob to be written, in bytes. It is implementation dependent whether * this value is used in writing the blob to the repository. * @param failIfAlreadyExists * whether to throw a FileAlreadyExistsException if the given blob already exists * @throws FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists * @throws IOException if the input stream could not be read, or the target blob could not be written to. */ default void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize, boolean failIfAlreadyExists) throws IOException { writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); }
/** * Writes blob with resolving the blob name using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param name blob name */ public void write(T obj, BlobContainer blobContainer, String name) throws IOException { final String blobName = blobName(name); writeTo(obj, blobName, bytesArray -> { try (InputStream stream = bytesArray.streamInput()) { blobContainer.writeBlob(blobName, stream, bytesArray.length(), true); } }); }
/** * Reads blob content from the input stream and writes it to the container in a new blob with the given name, * using an atomic write operation if the implementation supports it. When the BlobContainer implementation * does not provide a specific implementation of writeBlobAtomic(String, InputStream, long), then * the {@link #writeBlob(String, InputStream, long, boolean)} method is used. * * This method assumes the container does not already contain a blob of the same blobName. If a blob by the * same name already exists, the operation will fail and an {@link IOException} will be thrown. * * @param blobName * The name of the blob to write the contents of the input stream to. * @param inputStream * The input stream from which to retrieve the bytes to write to the blob. * @param blobSize * The size of the blob to be written, in bytes. It is implementation dependent whether * this value is used in writing the blob to the repository. * @param failIfAlreadyExists * whether to throw a FileAlreadyExistsException if the given blob already exists * @throws FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists * @throws IOException if the input stream could not be read, or the target blob could not be written to. */ default void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize, boolean failIfAlreadyExists) throws IOException { writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); }
/** * Reads blob content from the input stream and writes it to the container in a new blob with the given name, * using an atomic write operation if the implementation supports it. When the BlobContainer implementation * does not provide a specific implementation of writeBlobAtomic(String, InputStream, long), then * the {@link #writeBlob(String, InputStream, long, boolean)} method is used. * * This method assumes the container does not already contain a blob of the same blobName. If a blob by the * same name already exists, the operation will fail and an {@link IOException} will be thrown. * * @param blobName * The name of the blob to write the contents of the input stream to. * @param inputStream * The input stream from which to retrieve the bytes to write to the blob. * @param blobSize * The size of the blob to be written, in bytes. It is implementation dependent whether * this value is used in writing the blob to the repository. * @param failIfAlreadyExists * whether to throw a FileAlreadyExistsException if the given blob already exists * @throws FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists * @throws IOException if the input stream could not be read, or the target blob could not be written to. */ default void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize, boolean failIfAlreadyExists) throws IOException { writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); }
/** * Snapshot individual file * <p> * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are * added to the {@code failures} list * * @param fileInfo file to be snapshotted */ private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException { final String file = fileInfo.physicalName(); try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) { for (int i = 0; i < fileInfo.numberOfParts(); i++) { final long partBytes = fileInfo.partBytes(i); final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes); InputStream inputStream = inputStreamIndexInput; if (snapshotRateLimiter != null) { inputStream = new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotRateLimitingTimeInNanos::inc); } inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName()); blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes, true); } Store.verify(indexInput); snapshotStatus.addProcessedFile(fileInfo.length()); } catch (Exception t) { failStoreIfCorrupted(t); snapshotStatus.addProcessedFile(0); throw t; } }
protected static void writeBlob(BlobContainer container, String blobName, BytesArray bytesArray) throws IOException { try (InputStream stream = bytesArray.streamInput()) { container.writeBlob(blobName, stream, bytesArray.length()); } }
protected void writeBlob(final BlobContainer container, final String blobName, final BytesArray bytesArray) throws IOException { try (InputStream stream = bytesArray.streamInput()) { container.writeBlob(blobName, stream, bytesArray.length()); } }
@Override public void verify(String seed, DiscoveryNode localNode) { assertSnapshotOrGenericThread(); if (isReadOnly()) { try { latestIndexBlobId(); } catch (IOException e) { throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on node " + localNode, e); } } else { BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); if (testBlobContainer.blobExists("master.dat")) { try { BytesArray bytes = new BytesArray(seed); try (InputStream stream = bytes.streamInput()) { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true); } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } } }
/** * Writes blob with resolving the blob name using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param name blob name */ public void write(T obj, BlobContainer blobContainer, String name) throws IOException { final String blobName = blobName(name); writeTo(obj, blobName, bytesArray -> { try (InputStream stream = bytesArray.streamInput()) { blobContainer.writeBlob(blobName, stream, bytesArray.length(), true); } }); }
/** * Writes blob with resolving the blob name using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param name blob name */ public void write(T obj, BlobContainer blobContainer, String name) throws IOException { final String blobName = blobName(name); writeTo(obj, blobName, bytesArray -> { try (InputStream stream = bytesArray.streamInput()) { blobContainer.writeBlob(blobName, stream, bytesArray.length(), true); } }); }
/** * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param blobName blob name */ protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException { BytesReference bytes = write(obj); try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")"; try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, byteArrayOutputStream, BUFFER_SIZE)) { CodecUtil.writeHeader(indexOutput, codec, VERSION); try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) { @Override public void close() throws IOException { // this is important since some of the XContentBuilders write bytes on close. // in order to write the footer we need to prevent closing the actual index input. } }) { bytes.writeTo(indexOutputOutputStream); } CodecUtil.writeFooter(indexOutput); } blobContainer.writeBlob(blobName, new BytesArray(byteArrayOutputStream.toByteArray())); } }
private void writeAtomic(final String blobName, final BytesReference bytesRef) throws IOException { final String tempBlobName = "pending-" + blobName + "-" + UUIDs.randomBase64UUID(); try (InputStream stream = bytesRef.streamInput()) { snapshotsBlobContainer.writeBlob(tempBlobName, stream, bytesRef.length()); snapshotsBlobContainer.move(tempBlobName, blobName); } catch (IOException ex) { // temporary blob creation or move failed - try cleaning up try { snapshotsBlobContainer.deleteBlob(tempBlobName); } catch (IOException e) { ex.addSuppressed(e); } throw ex; } }
@Override public void verify(String seed) { BlobContainer testBlobContainer = blobStore.blobContainer(basePath.add(testBlobPrefix(seed))); DiscoveryNode localNode = clusterService.localNode(); if (testBlobContainer.blobExists("master.dat")) { try { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed)); } catch (IOException exp) { throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } }
@Override public String startVerification() { try { if (readOnly()) { // It's readonly - so there is not much we can do here to verify it return null; } else { String seed = Strings.randomBase64UUID(); byte[] testBytes = Strings.toUTF8Bytes(seed); BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); String blobName = "master.dat"; testContainer.writeBlob(blobName + "-temp", new BytesArray(testBytes)); // Make sure that move is supported testContainer.move(blobName + "-temp", blobName); return seed; } } catch (IOException exp) { throw new RepositoryVerificationException(repositoryName, "path " + basePath() + " is not accessible on master node", exp); } }
@Override public String startVerification() { try { if (isReadOnly()) { // It's readonly - so there is not much we can do here to verify it return null; } else { String seed = UUIDs.randomBase64UUID(); byte[] testBytes = Strings.toUTF8Bytes(seed); BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); String blobName = "master.dat"; BytesArray bytes = new BytesArray(testBytes); try (InputStream stream = bytes.streamInput()) { testContainer.writeBlob(blobName + "-temp", stream, bytes.length()); } // Make sure that move is supported testContainer.move(blobName + "-temp", blobName); return seed; } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on master node", exp); } }
/** * Snapshot individual file * <p> * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are * added to the {@code failures} list * * @param fileInfo file to be snapshotted */ private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException { final String file = fileInfo.physicalName(); try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) { for (int i = 0; i < fileInfo.numberOfParts(); i++) { final long partBytes = fileInfo.partBytes(i); final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes); InputStream inputStream = snapshotRateLimiter == null ? inputStreamIndexInput : new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotThrottleListener); inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName()); blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes); } Store.verify(indexInput); snapshotStatus.addProcessedFile(fileInfo.length()); } catch (Throwable t) { failStoreIfCorrupted(t); snapshotStatus.addProcessedFile(0); throw t; } }
@Override public void verify(String seed, DiscoveryNode localNode) { BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); if (testBlobContainer.blobExists("master.dat")) { try { BytesArray bytes = new BytesArray(seed); try (InputStream stream = bytes.streamInput()) { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length()); } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } }
@Override public void verify(String seed, DiscoveryNode localNode) { assertSnapshotOrGenericThread(); BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); if (testBlobContainer.blobExists("master.dat")) { try { BytesArray bytes = new BytesArray(seed); try (InputStream stream = bytes.streamInput()) { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true); } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } }
/** * Snapshot individual file * <p> * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are * added to the {@code failures} list * * @param fileInfo file to be snapshotted */ private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException { final String file = fileInfo.physicalName(); try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) { for (int i = 0; i < fileInfo.numberOfParts(); i++) { final long partBytes = fileInfo.partBytes(i); final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes); InputStream inputStream = inputStreamIndexInput; if (snapshotRateLimiter != null) { inputStream = new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotRateLimitingTimeInNanos::inc); } inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName()); blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes, true); } Store.verify(indexInput); snapshotStatus.addProcessedFile(fileInfo.length()); } catch (Exception t) { failStoreIfCorrupted(t); snapshotStatus.addProcessedFile(0); throw t; } }
@Override public void verify(String seed, DiscoveryNode localNode) { assertSnapshotOrGenericThread(); BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); if (testBlobContainer.blobExists("master.dat")) { try { BytesArray bytes = new BytesArray(seed); try (InputStream stream = bytes.streamInput()) { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true); } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } }