/** * Set our selves up for the next 'next' invocation, set up next block. * @return True is more to read else false if at the end. * @throws IOException */ private boolean positionForNextBlock() throws IOException { // Methods are small so they get inlined because they are 'hot'. long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (this.curBlock.getOffset() >= lastDataBlockOffset) { setNonSeekedState(); return false; } return isNextBlock(); }
FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset(); HFileBlock block; while (offset <= max) {
private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) throws IOException { BlockType.TRAILER.write(output); output.writeLong(fft.getFileInfoOffset()); output.writeLong(fft.getLoadOnOpenDataOffset()); output.writeInt(fft.getDataIndexCount()); output.writeLong(fft.getUncompressedDataIndexSize()); output.writeInt(fft.getMetaIndexCount()); output.writeLong(fft.getTotalUncompressedBytes()); output.writeLong(fft.getEntryCount()); output.writeInt(fft.getCompressionCodec().ordinal()); output.writeInt(fft.getNumDataIndexLevels()); output.writeLong(fft.getFirstDataBlockOffset()); output.writeLong(fft.getLastDataBlockOffset()); Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); }
justification="Yeah, unnecessary null check; could do w/ clean up") protected HFileBlock readNextDataBlock() throws IOException { long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (curBlock == null) return null;
/** * Read all blocks from {@code path} to populate {@code blockCache}. */ private static void cacheBlocks(Configuration conf, CacheConfig cacheConfig, FileSystem fs, Path path, HFileContext cxt) throws IOException { FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path); long fileSize = fs.getFileStatus(path).getLen(); FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); HFile.Reader reader = new HFileReaderImpl(path, trailer, fsdis, fileSize, cacheConfig, fsdis.getHfs(), conf); reader.loadFileInfo(); long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset(); List<HFileBlock> blocks = new ArrayList<>(4); HFileBlock block; while (offset <= max) { block = reader.readBlock(offset, -1, /* cacheBlock */ true, /* pread */ false, /* isCompaction */ false, /* updateCacheMetrics */ true, null, null); offset += block.getOnDiskSizeWithHeader(); blocks.add(block); } LOG.info("read " + Iterables.toString(blocks)); }
if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) { throw new IOException("Requested block is out of range: " + dataBlockOffset + ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset() + ", trailer.getLoadOnOpenDataOffset: " + trailerOffset);
private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) throws IOException { assertEquals(version, loaded.getMajorVersion()); assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); assertEquals(Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), loaded.getEntryCount()); if (version == 1) { assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); } if (version == 2) { assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); assertEquals(expected.createComparator().getClass().getName(), loaded.createComparator().getClass().getName()); assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( expected.createComparator() instanceof CellComparatorImpl); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); }
while (curBlockPos <= trailer.getLastDataBlockOffset()) { HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader);
/** * Set our selves up for the next 'next' invocation, set up next block. * @return True is more to read else false if at the end. * @throws IOException */ private boolean positionForNextBlock() throws IOException { // Methods are small so they get inlined because they are 'hot'. long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (block.getOffset() >= lastDataBlockOffset) { setNonSeekedState(); return false; } return isNextBlock(); }
reader.getTrailer().getLastDataBlockOffset();
FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset(); HFileBlock block; while (offset <= max) {
/** * Scans blocks in the "scanned" section of the {@link HFile} until the next * data block is found. * * @return the next block, or null if there are no more data blocks * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH", justification="Yeah, unnecessary null check; could do w/ clean up") protected HFileBlock readNextDataBlock() throws IOException { long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (block == null) return null; HFileBlock curBlock = block; do { if (curBlock.getOffset() >= lastDataBlockOffset) { return null; } if (curBlock.getOffset() < 0) { throw new IOException("Invalid block file offset: " + block); } // We are reading the next block without block type validation, because // it might turn out to be a non-data block. curBlock = reader.readBlock(curBlock.getOffset() + curBlock.getOnDiskSizeWithHeader(), curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread, isCompaction, true, null, getEffectiveDataBlockEncoding()); } while (!curBlock.getBlockType().isData()); return curBlock; }
private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) throws IOException { BlockType.TRAILER.write(output); output.writeLong(fft.getFileInfoOffset()); output.writeLong(fft.getLoadOnOpenDataOffset()); output.writeInt(fft.getDataIndexCount()); output.writeLong(fft.getUncompressedDataIndexSize()); output.writeInt(fft.getMetaIndexCount()); output.writeLong(fft.getTotalUncompressedBytes()); output.writeLong(fft.getEntryCount()); output.writeInt(fft.getCompressionCodec().ordinal()); output.writeInt(fft.getNumDataIndexLevels()); output.writeLong(fft.getFirstDataBlockOffset()); output.writeLong(fft.getLastDataBlockOffset()); Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); }
/** * Read all blocks from {@code path} to populate {@code blockCache}. */ private static void cacheBlocks(Configuration conf, CacheConfig cacheConfig, FileSystem fs, Path path, HFileContext cxt) throws IOException { FSDataInputStreamWrapper fsdis = new FSDataInputStreamWrapper(fs, path); long fileSize = fs.getFileStatus(path).getLen(); FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis.getStream(false), fileSize); HFile.Reader reader = new HFileReaderImpl(path, trailer, fsdis, fileSize, cacheConfig, fsdis.getHfs(), conf); reader.loadFileInfo(); long offset = trailer.getFirstDataBlockOffset(), max = trailer.getLastDataBlockOffset(); List<HFileBlock> blocks = new ArrayList<>(4); HFileBlock block; while (offset <= max) { block = reader.readBlock(offset, -1, /* cacheBlock */ true, /* pread */ false, /* isCompaction */ false, /* updateCacheMetrics */ true, null, null); offset += block.getOnDiskSizeWithHeader(); blocks.add(block); } LOG.info("read " + Iterables.toString(blocks)); }
/** * Scans blocks in the "scanned" section of the {@link HFile} until the next * data block is found. * * @return the next block, or null if there are no more data blocks * @throws IOException */ protected HFileBlock readNextDataBlock() throws IOException { long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset(); if (block == null) return null; HFileBlock curBlock = block; do { if (curBlock.getOffset() >= lastDataBlockOffset) return null; if (curBlock.getOffset() < 0) { throw new IOException("Invalid block file offset: " + block); } // We are reading the next block without block type validation, because // it might turn out to be a non-data block. curBlock = reader.readBlock(curBlock.getOffset() + curBlock.getOnDiskSizeWithHeader(), curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread, isCompaction, null); } while (!(curBlock.getBlockType().equals(BlockType.DATA) || curBlock.getBlockType().equals(BlockType.ENCODED_DATA))); return curBlock; } }
throw new IOException("Requested block is out of range: " + dataBlockOffset + ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset());
if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) { throw new IOException("Requested block is out of range: " + dataBlockOffset + ", lastDataBlockOffset: " + trailer.getLastDataBlockOffset() + ", trailer.getLoadOnOpenDataOffset: " + trailerOffset);
private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) throws IOException { assertEquals(version, loaded.getMajorVersion()); assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); assertEquals(Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), loaded.getEntryCount()); if (version == 1) { assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); } if (version == 2) { assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); assertEquals(expected.createComparator().getClass().getName(), loaded.createComparator().getClass().getName()); assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( expected.createComparator() instanceof CellComparatorImpl); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); }
while (curBlockPos <= trailer.getLastDataBlockOffset()) { HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader);