/** * File length * * @return file length */ public long length() { return metadata.length(); }
LuceneVerifyingIndexOutput(StoreFileMetaData metadata, IndexOutput out) { super(out); this.metadata = metadata; checksumPosition = metadata.length() - 8; // the last 8 bytes are the checksum - we store it in footerChecksum }
public long length() { return metaData.length(); }
/** * Returns total size of all files that where snapshotted */ public long totalSize() { return indexFiles.stream().mapToLong(fi -> fi.metadata().length()).sum(); }
/** * Constructs a new instance of file info * * @param name file name as stored in the blob store * @param metaData the files meta data * @param partSize size of the single chunk */ public FileInfo(String name, StoreFileMetaData metaData, ByteSizeValue partSize) { this.name = name; this.metadata = metaData; long partBytes = Long.MAX_VALUE; if (partSize != null && partSize.getBytes() > 0) { partBytes = partSize.getBytes(); } long totalLength = metaData.length(); long numberOfParts = totalLength / partBytes; if (totalLength % partBytes > 0) { numberOfParts++; } if (numberOfParts == 0) { numberOfParts++; } this.numberOfParts = numberOfParts; this.partSize = partSize; this.partBytes = partBytes; }
@Override public void writeByte(byte b) throws IOException { final long writtenBytes = this.writtenBytes++; if (writtenBytes >= checksumPosition) { // we are writing parts of the checksum.... if (writtenBytes == checksumPosition) { readAndCompareChecksum(); } final int index = Math.toIntExact(writtenBytes - checksumPosition); if (index < footerChecksum.length) { footerChecksum[index] = b; if (index == footerChecksum.length-1) { verify(); // we have recorded the entire checksum } } else { verify(); // fail if we write more than expected throw new AssertionError("write past EOF expected length: " + metadata.length() + " writtenBytes: " + writtenBytes); } } out.writeByte(b); }
@Override public void verify() throws IOException { String footerDigest = null; if (metadata.checksum().equals(actualChecksum) && writtenBytes == metadata.length()) { ByteArrayIndexInput indexInput = new ByteArrayIndexInput("checksum", this.footerChecksum); footerDigest = digestToString(indexInput.readLong()); if (metadata.checksum().equals(footerDigest)) { return; } } throw new CorruptIndexException("verification failed (hardware problem?) : expected=" + metadata.checksum() + " actual=" + actualChecksum + " footer=" + footerDigest +" writtenLength=" + writtenBytes + " expectedLength=" + metadata.length() + " (resource=" + metadata.toString() + ")", "VerifyingIndexOutput(" + metadata.name() + ")"); }
public static void checkIntegrity(final StoreFileMetaData md, final Directory directory) throws IOException { try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) { if (input.length() != md.length()) { // first check the length no matter how old this file is throw new CorruptIndexException("expected length=" + md.length() + " != actual length: " + input.length() + " : file truncated?", input); } // throw exception if the file is corrupt String checksum = Store.digestToString(CodecUtil.checksumEntireFile(input)); // throw exception if metadata is inconsistent if (!checksum.equals(md.checksum())) { throw new CorruptIndexException("inconsistent metadata: lucene checksum=" + checksum + ", metadata checksum=" + md.checksum(), input); } } }
private static long computeMatchingBytes(TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore, TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData) { String primarySyncId = primaryStore.syncId(); String replicaSyncId = storeFilesMetaData.syncId(); // see if we have a sync id we can make use of if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) { return Long.MAX_VALUE; } else { long sizeMatched = 0; for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) { String metaDataFileName = storeFileMetaData.name(); if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) { sizeMatched += storeFileMetaData.length(); } } return sizeMatched; } }
for (StoreFileMetaData md : diff.identical) { response.phase1ExistingFileNames.add(md.name()); response.phase1ExistingFileSizes.add(md.length()); existingTotalSize += md.length(); if (logger.isTraceEnabled()) { logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," + " size [{}]", md.name(), md.checksum(), md.length()); totalSize += md.length(); response.phase1FileSizes.add(md.length()); totalSize += md.length();
Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStreamFactory.apply(md)); } catch (Exception e) { final IOException corruptIndexException;
/** * File length * * @return file length */ public long length() { return metadata.length(); }
/** * Serializes file info into JSON * * @param file file info * @param builder XContent builder * @param params parameters */ public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(NAME, file.name); builder.field(PHYSICAL_NAME, file.metadata.name()); builder.field(LENGTH, file.metadata.length()); if (file.metadata.checksum().equals(UNKNOWN_CHECKSUM) == false) { builder.field(CHECKSUM, file.metadata.checksum()); } if (file.partSize != null) { builder.field(PART_SIZE, file.partSize.getBytes()); } if (file.metadata.writtenBy() != null) { builder.field(WRITTEN_BY, file.metadata.writtenBy()); } if (file.metadata.hash() != null && file.metadata().hash().length > 0) { BytesRef br = file.metadata.hash(); builder.field(META_HASH, br.bytes, br.offset, br.length); } builder.endObject(); }
if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) { try { Store.verify(indexOutput);
@Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeLong(recoveryId); shardId.writeTo(out); out.writeString(metaData.name()); out.writeVLong(position); out.writeVLong(metaData.length()); out.writeString(metaData.checksum()); out.writeBytesReference(content); out.writeString(metaData.writtenBy().toString()); out.writeBoolean(lastChunk); out.writeVInt(totalTranslogOps); out.writeLong(sourceThrottleTimeInNanos); }
/** * Returns total size of all files that where snapshotted */ public long totalSize() { return indexFiles.stream().mapToLong(fi -> fi.metadata().length()).sum(); }