@Override public long length() { return main.length(); }
/** Initializes the reader, for reuse on a new term. */ public void init(long skipPointer, int df) throws IOException { this.skipPointer[0] = skipPointer; this.docCount = df; assert skipPointer >= 0 && skipPointer <= skipStream[0].length() : "invalid skip pointer: " + skipPointer + ", length=" + skipStream[0].length(); Arrays.fill(skipDoc, 0); Arrays.fill(numSkipped, 0); Arrays.fill(childPointer, 0); for (int i = 1; i < numberOfSkipLevels; i++) { skipStream[i] = null; } loadSkipLevels(); }
private TermIterator(long delGen, RAMFile buffer) { try { input = new RAMInputStream("PrefixCodedTermsIterator", buffer); } catch (IOException e) { throw new RuntimeException(e); } end = input.length(); this.delGen = delGen; }
SlicedIndexInput(String sliceDescription, IndexInput base, long offset, long length) { super((sliceDescription == null) ? base.toString() : (base.toString() + " [slice=" + sliceDescription + "]"), BufferedIndexInput.BUFFER_SIZE); if (offset < 0 || length < 0 || offset + length > base.length()) { throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + base); } this.base = base.clone(); this.fileOffset = offset; this.length = length; }
/** * Returns (but does not validate) the checksum previously written by {@link #checkFooter}. * @return actual checksum value * @throws IOException if the footer is invalid */ public static long retrieveChecksum(IndexInput in) throws IOException { if (in.length() < footerLength()) { throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); } in.seek(in.length() - footerLength()); validateFooter(in); return readCRC(in); }
/** Retrieves the full footer from the provided {@link IndexInput}. This throws * {@link CorruptIndexException} if this file does not have a valid footer. */ public static byte[] readFooter(IndexInput in) throws IOException { if (in.length() < footerLength()) { throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); } in.seek(in.length() - footerLength()); validateFooter(in); in.seek(in.length() - footerLength()); byte[] bytes = new byte[footerLength()]; in.readBytes(bytes, 0, bytes.length); return bytes; }
/** Seek {@code input} to the directory offset. */ private static void seekDir(IndexInput input) throws IOException { input.seek(input.length() - CodecUtil.footerLength() - 8); long offset = input.readLong(); input.seek(offset); }
/** * Copies an existing {@code src} file from directory {@code from} * to a non-existent file {@code dest} in this directory. */ public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { boolean success = false; try (IndexInput is = from.openInput(src, context); IndexOutput os = createOutput(dest, context)) { os.copyBytes(is, is.length()); success = true; } finally { if (!success) { IOUtils.deleteFilesIgnoringExceptions(this, dest); } } }
VerifyingIndexInput(IndexInput input, Checksum digest) { super("VerifyingIndexInput(" + input + ")"); this.input = input; this.digest = digest; checksumPosition = input.length() - 8; }
@Override public long length() { return input.length(); }
private void unCache(String fileName) throws IOException { // Only let one thread uncache at a time; this only // happens during commit() or close(): synchronized(uncacheLock) { if (VERBOSE) { System.out.println("nrtdir.unCache name=" + fileName); } if (!cache.fileNameExists(fileName)) { // Another thread beat us... return; } assert slowFileExists(in, fileName) == false: "fileName=" + fileName + " exists both in cache and in delegate"; final IOContext context = IOContext.DEFAULT; final IndexOutput out = in.createOutput(fileName, context); IndexInput in = null; try { in = cache.openInput(fileName, context); out.copyBytes(in, in.length()); } finally { IOUtils.close(in, out); } // Lock order: uncacheLock -> this synchronized(this) { // Must sync here because other sync methods have // if (cache.fileNameExists(name)) { ... } else { ... }: cache.deleteFile(fileName); } } }
private static void validateFooter(IndexInput in) throws IOException { long remaining = in.length() - in.getFilePointer(); long expected = footerLength(); if (remaining < expected) { throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } else if (remaining > expected) { throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in); } final int magic = in.readInt(); if (magic != FOOTER_MAGIC) { throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in); } final int algorithmID = in.readInt(); if (algorithmID != 0) { throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in); } }
public static Checkpoint read(Path path) throws IOException { try (Directory dir = new SimpleFSDirectory(path.getParent())) { try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, INITIAL_VERSION, CURRENT_VERSION); if (fileVersion == INITIAL_VERSION) { assert indexInput.length() == V1_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV5_0_0(indexInput); } else if (fileVersion == VERSION_6_0_0) { assert indexInput.length() == V2_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV6_0_0(indexInput); } else { assert fileVersion == CURRENT_VERSION : fileVersion; assert indexInput.length() == V3_FILE_SIZE : indexInput.length(); return Checkpoint.readCheckpointV6_4_0(indexInput); } } } }
@Override public int read() throws IOException { if (counter++ >= limit) { return -1; } return (indexInput.getFilePointer() < indexInput.length()) ? (indexInput.readByte() & 0xff) : -1; }
@Override public int read(byte[] b, int off, int len) throws IOException { if (b == null) { throw new NullPointerException(); } else if (off < 0 || len < 0 || len > b.length - off) { throw new IndexOutOfBoundsException(); } if (indexInput.getFilePointer() >= indexInput.length()) { return -1; } if (indexInput.getFilePointer() + len > indexInput.length()) { len = (int) (indexInput.length() - indexInput.getFilePointer()); } if (counter + len > limit) { len = (int) (limit - counter); } if (len <= 0) { return -1; } indexInput.readBytes(b, off, len, false); counter += len; return len; }
public static void checkIntegrity(final StoreFileMetaData md, final Directory directory) throws IOException { try (IndexInput input = directory.openInput(md.name(), IOContext.READONCE)) { if (input.length() != md.length()) { // first check the length no matter how old this file is throw new CorruptIndexException("expected length=" + md.length() + " != actual length: " + input.length() + " : file truncated?", input); } // throw exception if the file is corrupt String checksum = Store.digestToString(CodecUtil.checksumEntireFile(input)); // throw exception if metadata is inconsistent if (!checksum.equals(md.checksum())) { throw new CorruptIndexException("inconsistent metadata: lucene checksum=" + checksum + ", metadata checksum=" + md.checksum(), input); } } }
private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { // additional safety we checksum the entire file we read the hash for... final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug(() -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }