private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); // additional safety we checksum the entire file we read the hash for... hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug((Supplier<?>) () -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }
private static void checksumFromLuceneFile(Directory directory, String file, ImmutableMap.Builder<String, StoreFileMetaData> builder, ESLogger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); // additional safety we checksum the entire file we read the hash for... hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Throwable ex) { logger.debug("Can retrieve checksum from file [{}]", ex, file); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }
private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); // additional safety we checksum the entire file we read the hash for... hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug(() -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }
private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); // additional safety we checksum the entire file we read the hash for... hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug(() -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }
@Override public void seek(long pos) throws IOException { if (pos < verifiedPosition) { // going within verified region - just seek there input.seek(pos); } else { if (verifiedPosition > getFilePointer()) { // portion of the skip region is verified and portion is not // skipping the verified portion input.seek(verifiedPosition); // and checking unverified skipBytes(pos - verifiedPosition); } else { skipBytes(pos - getFilePointer()); } } }
@Override public void seek(long pos) throws IOException { if (pos < verifiedPosition) { // going within verified region - just seek there input.seek(pos); } else { if (verifiedPosition > getFilePointer()) { // portion of the skip region is verified and portion is not // skipping the verified portion input.seek(verifiedPosition); // and checking unverified skipBytes(pos - verifiedPosition); } else { skipBytes(pos - getFilePointer()); } } }
@Override public void seek(long pos) throws IOException { if (pos < verifiedPosition) { // going within verified region - just seek there input.seek(pos); } else { if (verifiedPosition > getFilePointer()) { // portion of the skip region is verified and portion is not // skipping the verified portion input.seek(verifiedPosition); // and checking unverified skipBytes(pos - verifiedPosition); } else { skipBytes(pos - getFilePointer()); } } }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { assert metadata.writtenBy() != null; return new VerifyingIndexInput(directory().openInput(filename, context)); }
public static void verify(IndexOutput output) throws IOException { if (output instanceof VerifyingIndexOutput) { ((VerifyingIndexOutput) output).verify(); } }
public long verify() throws CorruptIndexException { long storedChecksum = getStoredChecksum(); if (getChecksum() == storedChecksum) { return storedChecksum; } throw new CorruptIndexException("verification failed : calculated=" + Store.digestToString(getChecksum()) + " stored=" + Store.digestToString(storedChecksum), this); }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { assert metadata.writtenBy() != null; return new VerifyingIndexInput(directory().openInput(filename, context)); }
public static void verify(IndexInput input) throws IOException { if (input instanceof VerifyingIndexInput) { ((VerifyingIndexInput) input).verify(); } }
public static void verify(IndexInput input) throws IOException { if (input instanceof VerifyingIndexInput) { ((VerifyingIndexInput) input).verify(); } }
public static void verify(IndexInput input) throws IOException { if (input instanceof VerifyingIndexInput) { ((VerifyingIndexInput) input).verify(); } }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { assert metadata.writtenBy() != null; assert metadata.writtenBy().onOrAfter(StoreFileMetaData.FIRST_LUCENE_CHECKSUM_VERSION); return new VerifyingIndexInput(directory().openInput(filename, context)); }
public long verify() throws CorruptIndexException { long storedChecksum = getStoredChecksum(); if (getChecksum() == storedChecksum) { return storedChecksum; } throw new CorruptIndexException("verification failed : calculated=" + Store.digestToString(getChecksum()) + " stored=" + Store.digestToString(storedChecksum), this); }
public IndexInput openVerifyingInput(String filename, IOContext context, StoreFileMetaData metadata) throws IOException { if (metadata.hasLegacyChecksum() || metadata.checksum() == null) { logger.debug("open legacy input for {}", filename); return directory().openInput(filename, context); } assert metadata.writtenBy() != null; assert metadata.writtenBy().onOrAfter(Version.LUCENE_4_8_0); return new VerifyingIndexInput(directory().openInput(filename, context)); }
public long verify() throws CorruptIndexException { long storedChecksum = getStoredChecksum(); if (getChecksum() == storedChecksum) { return storedChecksum; } throw new CorruptIndexException("verification failed : calculated=" + Store.digestToString(getChecksum()) + " stored=" + Store.digestToString(storedChecksum), this); }
public long verify() throws CorruptIndexException { long storedChecksum = getStoredChecksum(); if (getChecksum() == storedChecksum) { return storedChecksum; } throw new CorruptIndexException("verification failed : calculated=" + Store.digestToString(getChecksum()) + " stored=" + Store.digestToString(storedChecksum), this); }
private static void checksumFromLuceneFile(Directory directory, String file, Map<String, StoreFileMetaData> builder, Logger logger, Version version, boolean readFileAsHash) throws IOException { final String checksum; final BytesRefBuilder fileHash = new BytesRefBuilder(); try (IndexInput in = directory.openInput(file, IOContext.READONCE)) { final long length; try { length = in.length(); if (length < CodecUtil.footerLength()) { // truncated files trigger IAE if we seek negative... these files are really corrupted though throw new CorruptIndexException("Can't retrieve checksum from file: " + file + " file length must be >= " + CodecUtil.footerLength() + " but was: " + in.length(), in); } if (readFileAsHash) { // additional safety we checksum the entire file we read the hash for... final VerifyingIndexInput verifyingIndexInput = new VerifyingIndexInput(in); hashFile(fileHash, new InputStreamIndexInput(verifyingIndexInput, length), length); checksum = digestToString(verifyingIndexInput.verify()); } else { checksum = digestToString(CodecUtil.retrieveChecksum(in)); } } catch (Exception ex) { logger.debug(() -> new ParameterizedMessage("Can retrieve checksum from file [{}]", file), ex); throw ex; } builder.put(file, new StoreFileMetaData(file, length, checksum, version, fileHash.get())); } }