@Override public HFileBlock nextBlock() throws IOException { if (offset >= endOffset) { return null; } HFileBlock b = readBlockData(offset, length, false, false); offset += b.getOnDiskSizeWithHeader(); length = b.getNextBlockOnDiskSize(); return b.unpack(fileContext, owner); }
unpack(hfileContext, fsBlockReader);
fsBlockReader.readBlockData(dataBlockOffset, onDiskBlockSize, pread, !isCompaction); validateBlockType(hfileBlock, expectedBlockType); HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader); BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { HFileBlock compressedBlock = cachedBlock; cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader);
private long readAndVerifyBlock(long pos, HFileContext ctx, HFileBlock.FSReaderImpl hbr, int size) throws IOException { HFileBlock b = hbr.readBlockData(pos, -1, false, false); assertEquals(0, HFile.getAndResetChecksumFailuresCount()); b.sanityCheck(); assertFalse(b.isUnpacked()); b = b.unpack(ctx, hbr); LOG.info("Read a block at " + pos + " with" + " onDiskSizeWithHeader=" + b.getOnDiskSizeWithHeader() + " uncompressedSizeWithoutHeader=" + b.getOnDiskSizeWithoutHeader() + " uncompressedSizeWithoutHeader=" + b.getUncompressedSizeWithoutHeader()); DataInputStream dis = b.getByteStream(); for (int i = 0; i < size; i++) { int read = dis.readInt(); if (read != i) { fail("Block data corrupt at element " + i); } } return b.getOnDiskSizeWithHeader(); }
assertEquals(meta.isCompressedOrEncrypted(), !blockFromHFile.isUnpacked()); long packedHeapsize = blockFromHFile.heapSize(); blockUnpacked = blockFromHFile.unpack(meta, hbr); assertTrue(blockUnpacked.isUnpacked()); if (meta.isCompressedOrEncrypted()) { blockUnpacked, deserialized.unpack(meta, hbr));
b = b.unpack(meta, hbr);
while (curBlockPos <= trailer.getLastDataBlockOffset()) { HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader); assertEquals(BlockType.DATA, block.getBlockType()); ByteBuff buf = block.getBufferWithoutHeader(); trailer.getLoadOnOpenDataOffset()); HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader); assertEquals(BlockType.META, block.getBlockType()); Text t = new Text();
assertFalse(fromCache.isUnpacked()); fromCache = fromCache.unpack(meta, reader.getUncachedBlockReader()); } else { assertTrue(fromCache.isUnpacked());
b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); ByteBuff bb = b.unpack(meta, hbr).getBufferWithoutHeader(); // read back data DataInputStream in = new DataInputStream( new ByteArrayInputStream( is.close(); b.sanityCheck(); b = b.unpack(meta, hbr); assertEquals(4936, b.getUncompressedSizeWithoutHeader()); assertEquals(algo == GZ ? 2173 : 4936,
@Override public HFileBlock nextBlock() throws IOException { if (offset >= endOffset) return null; HFileBlock b = readBlockData(offset, -1, -1, false); offset += b.getOnDiskSizeWithHeader(); return b.unpack(fileContext, owner); }
private long readAndVerifyBlock(long pos, HFileContext ctx, HFileBlock.FSReaderImpl hbr, int size) throws IOException { HFileBlock b = hbr.readBlockData(pos, -1, false, false); assertEquals(0, HFile.getAndResetChecksumFailuresCount()); b.sanityCheck(); assertFalse(b.isUnpacked()); b = b.unpack(ctx, hbr); LOG.info("Read a block at " + pos + " with" + " onDiskSizeWithHeader=" + b.getOnDiskSizeWithHeader() + " uncompressedSizeWithoutHeader=" + b.getOnDiskSizeWithoutHeader() + " uncompressedSizeWithoutHeader=" + b.getUncompressedSizeWithoutHeader()); DataInputStream dis = b.getByteStream(); for (int i = 0; i < size; i++) { int read = dis.readInt(); if (read != i) { fail("Block data corrupt at element " + i); } } return b.getOnDiskSizeWithHeader(); }
blockSize, -1, true).unpack(hfileContext, fsBlockReader);
pread); validateBlockType(hfileBlock, expectedBlockType); HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader); BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory();
assertEquals(meta.isCompressedOrEncrypted(), !blockFromHFile.isUnpacked()); long packedHeapsize = blockFromHFile.heapSize(); blockUnpacked = blockFromHFile.unpack(meta, hbr); assertTrue(blockUnpacked.isUnpacked()); if (meta.isCompressedOrEncrypted()) { blockUnpacked, deserialized.unpack(meta, hbr));
if (cachedBlock != null) { if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { cachedBlock = cachedBlock.unpack(hfileContext, fsBlockReader);
b = b.unpack(meta, hbr);
while (curBlockPos <= trailer.getLastDataBlockOffset()) { HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader); assertEquals(BlockType.DATA, block.getBlockType()); ByteBuff buf = block.getBufferWithoutHeader(); trailer.getLoadOnOpenDataOffset()); HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader); assertEquals(BlockType.META, block.getBlockType()); Text t = new Text();
b.getOnDiskSizeWithoutHeader() - b.totalChecksumBytes()); ByteBuff bb = b.unpack(meta, hbr).getBufferWithoutHeader(); // read back data DataInputStream in = new DataInputStream( new ByteArrayInputStream( is.close(); b.sanityCheck(); b = b.unpack(meta, hbr); assertEquals(4936, b.getUncompressedSizeWithoutHeader()); assertEquals(algo == GZ ? 2173 : 4936,
assertFalse(fromCache.isUnpacked()); fromCache = fromCache.unpack(meta, reader.getUncachedBlockReader()); } else { assertTrue(fromCache.isUnpacked());