/** * Checks obj in the blob container */ public boolean exists(BlobContainer blobContainer, String name) throws IOException { return blobContainer.blobExists(blobName(name)); }
/** * Deletes a blob with giving name, ignoring if the blob does not exist. * * @param blobName * The name of the blob to delete. * @throws IOException if the blob exists but could not be deleted. */ default void deleteBlobIgnoringIfNotExists(String blobName) throws IOException { try { deleteBlob(blobName); } catch (final NoSuchFileException ignored) { // This exception is ignored } }
/** * Delete shard snapshot */ public void delete() { final Map<String, BlobMetaData> blobs; try { blobs = blobContainer.listBlobs(); } catch (IOException e) { throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e); } Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs); BlobStoreIndexShardSnapshots snapshots = tuple.v1(); int fileListGeneration = tuple.v2(); try { indexShardSnapshotFormat.delete(blobContainer, snapshotId.getUUID()); } catch (IOException e) { logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId); } // Build a list of snapshots that should be preserved List<SnapshotFiles> newSnapshotsList = new ArrayList<>(); for (SnapshotFiles point : snapshots) { if (!point.snapshot().equals(snapshotId.getName())) { newSnapshotsList.add(point); } } // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index finalize(newSnapshotsList, fileListGeneration + 1, blobs, "snapshot deletion [" + snapshotId + "]"); }
@Override protected InputStream openSlice(long slice) throws IOException { return container.readBlob(info.partName(slice)); } }
@Override public void verify(String seed, DiscoveryNode localNode) { assertSnapshotOrGenericThread(); if (isReadOnly()) { try { latestIndexBlobId(); } catch (IOException e) { throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on node " + localNode, e); } } else { BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); if (testBlobContainer.blobExists("master.dat")) { try { BytesArray bytes = new BytesArray(seed); try (InputStream stream = bytes.streamInput()) { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length(), true); } } catch (IOException exp) { throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } } }
/** * Writes snapshot index file * <p> * This file can be used by read-only repositories that are unable to list files in the repository * * @param snapshots list of snapshot ids * @throws IOException I/O errors */ protected void writeSnapshotList(List<SnapshotId> snapshots) throws IOException { final BytesReference bRef; try(BytesStreamOutput bStream = new BytesStreamOutput()) { try(StreamOutput stream = new OutputStreamStreamOutput(bStream)) { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); builder.startObject(); builder.startArray("snapshots"); for (SnapshotId snapshot : snapshots) { builder.value(snapshot.getSnapshot()); } builder.endArray(); builder.endObject(); builder.close(); } bRef = bStream.bytes(); } if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) { snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE); } snapshotsBlobContainer.writeBlob(SNAPSHOTS_FILE, bRef); }
private void writeAtomic(final String blobName, final BytesReference bytesRef) throws IOException { final String tempBlobName = "pending-" + blobName + "-" + UUIDs.randomBase64UUID(); try (InputStream stream = bytesRef.streamInput()) { snapshotsBlobContainer.writeBlob(tempBlobName, stream, bytesRef.length()); snapshotsBlobContainer.move(tempBlobName, blobName); } catch (IOException ex) { // temporary blob creation or move failed - try cleaning up try { snapshotsBlobContainer.deleteBlob(tempBlobName); } catch (IOException e) { ex.addSuppressed(e); } throw ex; } }
/** * Reads blob content from the input stream and writes it to the container in a new blob with the given name, * using an atomic write operation if the implementation supports it. When the BlobContainer implementation * does not provide a specific implementation of writeBlobAtomic(String, InputStream, long), then * the {@link #writeBlob(String, InputStream, long, boolean)} method is used. * * This method assumes the container does not already contain a blob of the same blobName. If a blob by the * same name already exists, the operation will fail and an {@link IOException} will be thrown. * * @param blobName * The name of the blob to write the contents of the input stream to. * @param inputStream * The input stream from which to retrieve the bytes to write to the blob. * @param blobSize * The size of the blob to be written, in bytes. It is implementation dependent whether * this value is used in writing the blob to the repository. * @param failIfAlreadyExists * whether to throw a FileAlreadyExistsException if the given blob already exists * @throws FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists * @throws IOException if the input stream could not be read, or the target blob could not be written to. */ default void writeBlobAtomic(final String blobName, final InputStream inputStream, final long blobSize, boolean failIfAlreadyExists) throws IOException { writeBlob(blobName, inputStream, blobSize, failIfAlreadyExists); }
public void testMoveAndList() throws IOException { try(BlobStore store = newBlobStore()) { final BlobContainer container = store.blobContainer(new BlobPath()); assertThat(container.listBlobs().size(), equalTo(0)); int numberOfFooBlobs = randomIntBetween(0, 10); int numberOfBarBlobs = randomIntBetween(3, 20); byte[] data = writeRandomBlob(container, name, length); Map<String, BlobMetaData> blobs = container.listBlobs(); assertThat(blobs.size(), equalTo(numberOfFooBlobs + numberOfBarBlobs)); for (Map.Entry<String, Long> generated : generatedBlobs.entrySet()) { assertThat(container.listBlobsByPrefix("foo-").size(), equalTo(numberOfFooBlobs)); assertThat(container.listBlobsByPrefix("bar-").size(), equalTo(numberOfBarBlobs)); assertThat(container.listBlobsByPrefix("baz-").size(), equalTo(0)); container.move(name, newName); assertThat(container.listBlobsByPrefix(name).size(), equalTo(0)); blobs = container.listBlobsByPrefix(newName); assertThat(blobs.size(), equalTo(1)); assertThat(blobs.get(newName).length(), equalTo(generatedBlobs.get(name)));
/** * Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository. * * Package private for testing. */ void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException { assert isReadOnly() == false; // can not write to a read only repository final BytesReference bytes; try (BytesStreamOutput bStream = new BytesStreamOutput()) { try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream); repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS); builder.close(); } bytes = bStream.bytes(); } if (snapshotsBlobContainer.blobExists(INCOMPATIBLE_SNAPSHOTS_BLOB)) { snapshotsBlobContainer.deleteBlob(INCOMPATIBLE_SNAPSHOTS_BLOB); } // write the incompatible snapshots blob writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes); }
/** * Writes blob in atomic manner with resolving the blob name using {@link #blobName} and {@link #tempBlobName} methods. * <p> * The blob will be compressed and checksum will be written if required. * * Atomic move might be very inefficient on some repositories. It also cannot override existing files. * * @param obj object to be serialized * @param blobContainer blob container * @param name blob name */ public void writeAtomic(T obj, BlobContainer blobContainer, String name) throws IOException { String blobName = blobName(name); String tempBlobName = tempBlobName(name); writeBlob(obj, blobContainer, tempBlobName); try { blobContainer.move(tempBlobName, blobName); } catch (IOException ex) { // Move failed - try cleaning up blobContainer.deleteBlob(tempBlobName); throw ex; } }
@Override public String startVerification() { try { if (readOnly()) { // It's readonly - so there is not much we can do here to verify it return null; } else { String seed = Strings.randomBase64UUID(); byte[] testBytes = Strings.toUTF8Bytes(seed); BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed))); String blobName = "master.dat"; testContainer.writeBlob(blobName + "-temp", new BytesArray(testBytes)); // Make sure that move is supported testContainer.move(blobName + "-temp", blobName); return seed; } } catch (IOException exp) { throw new RepositoryVerificationException(repositoryName, "path " + basePath() + " is not accessible on master node", exp); } }
private long listBlobsToGetLatestIndexId() throws IOException { Map<String, BlobMetaData> blobs = blobContainer().listBlobsByPrefix(INDEX_FILE_PREFIX); long latest = RepositoryData.EMPTY_REPO_GEN; if (blobs.isEmpty()) { // no snapshot index blobs have been written yet return latest; } for (final BlobMetaData blobMetaData : blobs.values()) { final String blobName = blobMetaData.name(); try { final long curr = Long.parseLong(blobName.substring(INDEX_FILE_PREFIX.length())); latest = Math.max(latest, curr); } catch (NumberFormatException nfe) { // the index- blob wasn't of the format index-N where N is a number, // no idea what this blob is but it doesn't belong in the repository! logger.debug("[{}] Unknown blob in the repository: {}", metadata.name(), blobName); } } return latest; }
logger.debug("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", blobContainer.path());
private void writeAtomic(final String blobName, final BytesReference bytesRef, boolean failIfAlreadyExists) throws IOException { try (InputStream stream = bytesRef.streamInput()) { blobContainer().writeBlobAtomic(blobName, stream, bytesRef.length(), failIfAlreadyExists); } }
if (FsBlobContainer.isTempBlobName(blobName)) { try { blobContainer.deleteBlobIgnoringIfNotExists(blobName); } catch (IOException e) { logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete index blob [{}] during finalization", if (blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) { try { blobContainer.deleteBlobIgnoringIfNotExists(blobName); } catch (IOException e) { logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete index blob [{}] during finalization", if (blobName.startsWith(DATA_BLOB_PREFIX) && (updatedSnapshots.findNameFile(canonicalName(blobName)) == null)) { try { blobContainer.deleteBlobIgnoringIfNotExists(blobName); } catch (IOException e) { logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to delete data blob [{}] during finalization",
long readSnapshotIndexLatestBlob() throws IOException { try (InputStream blob = blobContainer().readBlob(INDEX_LATEST_BLOB)) { BytesStreamOutput out = new BytesStreamOutput(); Streams.copy(blob, out); return Numbers.bytesToLong(out.bytes().toBytesRef()); } }
@Override public void verify(String seed) { BlobContainer testBlobContainer = blobStore.blobContainer(basePath.add(testBlobPrefix(seed))); DiscoveryNode localNode = clusterService.localNode(); if (testBlobContainer.blobExists("master.dat")) { try { testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed)); } catch (IOException exp) { throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore + "] is not accessible on the node [" + localNode + "]", exp); } } else { throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore + "] cannot be accessed on the node [" + localNode + "]. " + "This might indicate that the store [" + blobStore + "] is not shared between this node and the master node or " + "that permissions on the store don't allow reading files written by the master node"); } }
/** * Writes blob with resolving the blob name using {@link #blobName} method. * <p> * The blob will be compressed and checksum will be written if required. * * @param obj object to be serialized * @param blobContainer blob container * @param name blob name */ public void write(T obj, BlobContainer blobContainer, String name) throws IOException { final String blobName = blobName(name); writeTo(obj, blobName, bytesArray -> { try (InputStream stream = bytesArray.streamInput()) { blobContainer.writeBlob(blobName, stream, bytesArray.length(), true); } }); }
if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) { snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile); genBytes = bStream.bytes(); if (snapshotsBlobContainer.blobExists(INDEX_LATEST_BLOB)) { snapshotsBlobContainer.deleteBlob(INDEX_LATEST_BLOB);