Directory getSnapshotDirectory() { return new FilterDirectory(store.directory()) { @Override public String[] listAll() throws IOException {
public boolean checkIntegrityNoException(StoreFileMetaData md) { return checkIntegrityNoException(md, directory()); }
private IndexWriter createWriter() throws IOException { try { final IndexWriterConfig iwc = getIndexWriterConfig(); return createWriter(store.directory(), iwc); } catch (LockObtainFailedException ex) { logger.warn("could not lock IndexWriter", ex); throw ex; } }
private boolean assertSequenceNumbersInCommit() throws IOException { final Map<String, String> userData = SegmentInfos.readLatestCommit(store.directory()).getUserData(); assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint"; assert userData.containsKey(SequenceNumbers.MAX_SEQ_NO) : "commit point doesn't contains a maximum sequence number"; assert userData.containsKey(Engine.HISTORY_UUID_KEY) : "commit point doesn't contains a history uuid"; assert userData.get(Engine.HISTORY_UUID_KEY).equals(getHistoryUUID()) : "commit point history uuid [" + userData.get(Engine.HISTORY_UUID_KEY) + "] is different than engine [" + getHistoryUUID() + "]"; return true; }
public boolean isMarkedCorrupted() throws IOException { ensureOpen(); /* marking a store as corrupted is basically adding a _corrupted to all * the files. This prevent */ final String[] files = directory().listAll(); for (String file : files) { if (file.startsWith(CORRUPTED)) { return true; } } return false; }
private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException { final Directory directory = store.directory(); for (String name : Lucene.files(si)) { long length = directory.fileLength(name); index.addFileDetail(name, length, true); } }
private boolean assertMaxUnsafeAutoIdInCommit() throws IOException { final Map<String, String> userData = SegmentInfos.readLatestCommit(store.directory()).getUserData(); if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_5_0)) { // as of 5.5.0, the engine stores the maxUnsafeAutoIdTimestamp in the commit point. // This should have baked into the commit by the primary we recover from, regardless of the index age. assert userData.containsKey(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID) : "opening index which was created post 5.5.0 but " + Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID + " is not found in commit"; } return true; }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { assert metadata.writtenBy() != null; return new VerifyingIndexInput(directory().openInput(filename, context)); }
try (Lock writeLock = directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) { for (Map.Entry<String, String> entry : entries) { String tempFile = entry.getKey();
/** * Returns the last committed segments info for this store * * @throws IOException if the index is corrupted or the segments file is not present */ public SegmentInfos readLastCommittedSegmentsInfo() throws IOException { failIfCorrupted(); try { return readSegmentsInfo(null, directory()); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { markStoreCorrupted(ex); throw ex; } }
private void doCheckIndex() throws IOException { long timeNS = System.nanoTime(); if (!Lucene.indexExists(store.directory())) { return; for (Map.Entry<String, StoreFileMetaData> entry : metadata.asMap().entrySet()) { try { Store.checkIntegrity(entry.getValue(), store.directory()); out.println("checksum passed: " + entry.getKey()); } catch (IOException exc) {
/** * Deletes all corruption markers from this store. */ public void removeCorruptionMarker() throws IOException { ensureOpen(); final Directory directory = directory(); IOException firstException = null; final String[] files = directory.listAll(); for (String file : files) { if (file.startsWith(CORRUPTED)) { try { directory.deleteFile(file); } catch (IOException ex) { if (firstException == null) { firstException = ex; } else { firstException.addSuppressed(ex); } } } } if (firstException != null) { throw firstException; } }
/** * The returned IndexOutput validates the files checksum. * <p> * Note: Checksums are calculated by default since version 4.8.0. This method only adds the * verification against the checksum in the given metadata and does not add any significant overhead. */ public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException { IndexOutput output = directory().createOutput(fileName, context); boolean success = false; try { assert metadata.writtenBy() != null; output = new LuceneVerifyingIndexOutput(metadata, output); success = true; } finally { if (success == false) { IOUtils.closeWhileHandlingException(output); } } return output; }
logger.debug("starting recovery from local shards {}", shards); try { final Directory directory = indexShard.store().directory(); // don't close this directory!! final Directory[] sources = shards.stream().map(LocalShardSnapshot::getSnapshotDirectory).toArray(Directory[]::new); final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong();
/** * Marks this store as corrupted. This method writes a {@code corrupted_${uuid}} file containing the given exception * message. If a store contains a {@code corrupted_${uuid}} file {@link #isMarkedCorrupted()} will return <code>true</code>. */ public void markStoreCorrupted(IOException exception) throws IOException { ensureOpen(); if (!isMarkedCorrupted()) { String uuid = CORRUPTED + UUIDs.randomBase64UUID(); try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) { CodecUtil.writeHeader(output, CODEC, VERSION); BytesStreamOutput out = new BytesStreamOutput(); out.writeException(exception); BytesReference bytes = out.bytes(); output.writeVInt(bytes.length()); BytesRef ref = bytes.toBytesRef(); output.writeBytes(ref.bytes, ref.offset, ref.length); CodecUtil.writeFooter(output); } catch (IOException ex) { logger.warn("Can't mark store as corrupted", ex); } directory().sync(Collections.singleton(uuid)); } }
final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY); final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.translogLocation(), translogUUID); final List<IndexCommit> existingCommits = DirectoryReader.listCommits(store.directory()); final IndexCommit safeCommit = CombinedDeletionPolicy.findSafeCommitPoint(existingCommits, globalCheckpoint); final SequenceNumbers.CommitInfo seqNoStats = Store.loadSeqNoInfo(safeCommit);
assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll()); store.directory().sync(Collections.singleton(temporaryFileName)); IndexOutput remove = removeOpenIndexOutputs(name); assert remove == null || remove == indexOutput; // remove maybe null if we got finished
try (IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
store.directory().sync(Collections.singleton(fileInfo.physicalName())); success = true; } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
store.removeCorruptionMarker(); } finally { Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files