/** * Returns original file name * * @return original file name */ public String physicalName() { return metadata.name(); }
public String name() { return metaData.name(); }
private int numSegmentFiles() { // only for asserts int count = 0; for (StoreFileMetaData file : this) { if (file.name().startsWith(IndexFileNames.SEGMENTS)) { count++; } } return count; }
/** * Returns the segments file that this metadata snapshot represents or null if the snapshot is empty. */ public StoreFileMetaData getSegmentsFile() { for (StoreFileMetaData file : this) { if (file.name().startsWith(IndexFileNames.SEGMENTS)) { return file; } } assert metadata.isEmpty(); return null; }
final void verifyAfterCleanup(MetadataSnapshot sourceMetaData, MetadataSnapshot targetMetaData) { final RecoveryDiff recoveryDiff = targetMetaData.recoveryDiff(sourceMetaData); if (recoveryDiff.identical.size() != recoveryDiff.size()) { if (recoveryDiff.missing.isEmpty()) { for (StoreFileMetaData meta : recoveryDiff.different) { StoreFileMetaData local = targetMetaData.get(meta.name()); StoreFileMetaData remote = sourceMetaData.get(meta.name()); // if we have different files then they must have no checksums; otherwise something went wrong during recovery. // we have that problem when we have an empty index is only a segments_1 file so we can't tell if it's a Lucene 4.8 file // and therefore no checksum is included. That isn't a problem since we simply copy it over anyway but those files // come out as different in the diff. That's why we have to double check here again if the rest of it matches. // all is fine this file is just part of a commit or a segment that is different if (local.isSame(remote) == false) { logger.debug("Files are different on the recovery target: {} ", recoveryDiff); throw new IllegalStateException("local version: " + local + " is different from remote version after recovery: " + remote, null); } } } else { logger.debug("Files are missing on the recovery target: {} ", recoveryDiff); throw new IllegalStateException("Files are missing on the recovery target: [different=" + recoveryDiff.different + ", missing=" + recoveryDiff.missing + ']', null); } } }
if (IndexFileNames.OLD_SEGMENTS_GEN.equals(meta.name())) { // legacy continue; // we don't need that file at all final String segmentId = IndexFileNames.parseSegmentName(meta.name()); final String extension = IndexFileNames.getExtension(meta.name()); if (IndexFileNames.SEGMENTS.equals(segmentId) || DEL_FILE_EXTENSION.equals(extension) || LIV_FILE_EXTENSION.equals(extension)) { boolean consistent = true; for (StoreFileMetaData meta : segmentFiles) { StoreFileMetaData storeFileMetaData = recoveryTargetSnapshot.get(meta.name()); if (storeFileMetaData == null) { consistent = false;
/** * Read from a stream. */ public MetadataSnapshot(StreamInput in) throws IOException { final int size = in.readVInt(); Map<String, StoreFileMetaData> metadata = new HashMap<>(); for (int i = 0; i < size; i++) { StoreFileMetaData meta = new StoreFileMetaData(in); metadata.put(meta.name(), meta); } Map<String, String> commitUserData = new HashMap<>(); int num = in.readVInt(); for (int i = num; i > 0; i--) { commitUserData.put(in.readString(), in.readString()); } this.metadata = unmodifiableMap(metadata); this.commitUserData = unmodifiableMap(commitUserData); this.numDocs = in.readLong(); assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles(); }
private void readAndCompareChecksum() throws IOException { actualChecksum = digestToString(getChecksum()); if (!metadata.checksum().equals(actualChecksum)) { throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); } }
private static long computeMatchingBytes(TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore, TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData) { String primarySyncId = primaryStore.syncId(); String replicaSyncId = storeFilesMetaData.syncId(); // see if we have a sync id we can make use of if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) { return Long.MAX_VALUE; } else { long sizeMatched = 0; for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) { String metaDataFileName = storeFileMetaData.name(); if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) { sizeMatched += storeFileMetaData.length(); } } return sizeMatched; } }
@Override public void verify() throws IOException { String footerDigest = null; if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) { ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum); footerDigest = digestToString(indexInput.readLong()); if (metadata.checksum().equals(footerDigest)) { return; } } throw new CorruptIndexException("verification failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " footer=" + footerDigest +" writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); }
final Store.RecoveryDiff diff = recoverySourceMetadata.recoveryDiff(request.metadataSnapshot()); for (StoreFileMetaData md : diff.identical) { response.phase1ExistingFileNames.add(md.name()); response.phase1ExistingFileSizes.add(md.length()); existingTotalSize += md.length(); if (logger.isTraceEnabled()) { logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," + " size [{}]", md.name(), md.checksum(), md.length()); phase1Files.addAll(diff.missing); for (StoreFileMetaData md : phase1Files) { if (request.metadataSnapshot().asMap().containsKey(md.name())) { logger.trace("recovery [phase1]: recovering [{}], exists in local store, but is different: remote [{}], local [{}]", md.name(), request.metadataSnapshot().asMap().get(md.name()), md); } else { logger.trace("recovery [phase1]: recovering [{}], does not exist in remote", md.name()); response.phase1FileNames.add(md.name()); response.phase1FileSizes.add(md.length()); totalSize += md.length();
public static void checkIntegrity(final StoreFileMetaData md, final Directory directory) throws IOException { try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) { if (input.length() != md.length()) { // first check the length no matter how old this file is throw new CorruptIndexException("expected length=" + md.length() + " != actual length: " + input.length() + " : file truncated?", input); } // throw exception if the file is corrupt String checksum = Store.digestToString(CodecUtil.checksumEntireFile(input)); // throw exception if metadata is inconsistent if (!checksum.equals(md.checksum())) { throw new CorruptIndexException("inconsistent metadata: lucene checksum=" + checksum + ", metadata checksum=" + md.checksum(), input); } } }
try (IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
snapshotMetaData.put(fileInfo.metadata().name(), fileInfo.metadata()); fileInfos.put(fileInfo.metadata().name(), fileInfo); BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name()); recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), true); if (logger.isTraceEnabled()) { BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name()); filesToRecover.add(fileInfo); recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), false); segmentCommitInfos = Lucene.pruneUnreferencedFiles(restoredSegmentsFile.name(), store.directory()); } catch (IOException e) { throw new IndexShardRestoreFailedException(shardId, "Failed to fetch index version after copying it over", e);
/** * Returns original file name * * @return original file name */ public String physicalName() { return metadata.name(); }
/** * Serializes file info into JSON * * @param file file info * @param builder XContent builder * @param params parameters */ public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(NAME, file.name); builder.field(PHYSICAL_NAME, file.metadata.name()); builder.field(LENGTH, file.metadata.length()); if (file.metadata.checksum().equals(UNKNOWN_CHECKSUM) == false) { builder.field(CHECKSUM, file.metadata.checksum()); } if (file.partSize != null) { builder.field(PART_SIZE, file.partSize.getBytes()); } if (file.metadata.writtenBy() != null) { builder.field(WRITTEN_BY, file.metadata.writtenBy()); } if (file.metadata.hash() != null && file.metadata().hash().length > 0) { BytesRef br = file.metadata.hash(); builder.field(META_HASH, br.bytes, br.offset, br.length); } builder.endObject(); }
private int numSegmentFiles() { // only for asserts int count = 0; for (StoreFileMetaData file : this) { if (file.name().startsWith(IndexFileNames.SEGMENTS)) { count++; } } return count; }
boolean lastChunk, int totalTranslogOps) throws IOException { final Store store = store(); final String name = fileMetaData.name(); state().getTranslog().totalOperations(totalTranslogOps); final RecoveryState.Index indexState = state().getIndex();
@Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeLong(recoveryId); shardId.writeTo(out); out.writeString(metaData.name()); out.writeVLong(position); out.writeVLong(metaData.length()); out.writeString(metaData.checksum()); out.writeBytesReference(content); out.writeString(metaData.writtenBy().toString()); out.writeBoolean(lastChunk); out.writeVInt(totalTranslogOps); out.writeLong(sourceThrottleTimeInNanos); }
private void readAndCompareChecksum() throws IOException { actualChecksum = digestToString(getChecksum()); if (!metadata.checksum().equals(actualChecksum)) { throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); } }