public HFileBlock deepClone() { return new HFileBlock(this, true); } }
new HFileBlock(new SingleByteBuff(onDiskBlockByteBuffer), checksumSupport, MemoryType.EXCLUSIVE, offset, nextBlockOnDiskSize, fileContext);
@Test public void testSerializeWithoutNextBlockMetadata() { int size = 100; int length = HConstants.HFILEBLOCK_HEADER_SIZE + size; byte[] byteArr = new byte[length]; ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size); HFileContext meta = new HFileContextBuilder().build(); HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 52, -1, meta); HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, -1, -1, meta); ByteBuffer buff1 = ByteBuffer.allocate(length); ByteBuffer buff2 = ByteBuffer.allocate(length); blockWithNextBlockMetadata.serialize(buff1, true); blockWithoutNextBlockMetadata.serialize(buff2, true); assertNotEquals(buff1, buff2); buff1.clear(); buff2.clear(); blockWithNextBlockMetadata.serialize(buff1, false); blockWithoutNextBlockMetadata.serialize(buff2, false); assertEquals(buff1, buff2); } }
.withChecksumType(ChecksumType.NULL) .build(); HFileBlock generated = new HFileBlock(BlockType.DATA, onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader, prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) { ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE); buf.position(HConstants.HFILEBLOCK_HEADER_SIZE); keyValues.rewind(); buf.put(keyValues); HFileContext meta = new HFileContextBuilder() .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTag) .withHBaseCheckSum(true) .withCompression(Algorithm.NONE) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, meta); return b; }
private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException { int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM; // Create some KVs and create the block with old-style header. List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags); ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + headerSize); buf.position(headerSize); keyValues.rewind(); buf.put(keyValues); HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTags) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, hfileContext); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags); assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length); }
@Override public HFileBlock deserialize(ByteBuff buf, boolean reuse, MemoryType memType) throws IOException { // The buf has the file block followed by block metadata. // Set limit to just before the BLOCK_METADATA_SPACE then rewind. buf.limit(buf.limit() - BLOCK_METADATA_SPACE).rewind(); // Get a new buffer to pass the HFileBlock for it to 'own'. ByteBuff newByteBuff; if (reuse) { newByteBuff = buf.slice(); } else { int len = buf.limit(); newByteBuff = new SingleByteBuff(ByteBuffer.allocate(len)); newByteBuff.put(0, buf, buf.position(), len); } // Read out the BLOCK_METADATA_SPACE content and shove into our HFileBlock. buf.position(buf.limit()); buf.limit(buf.limit() + HFileBlock.BLOCK_METADATA_SPACE); boolean usesChecksum = buf.get() == (byte) 1; long offset = buf.getLong(); int nextBlockOnDiskSize = buf.getInt(); HFileBlock hFileBlock = new HFileBlock(newByteBuff, usesChecksum, memType, offset, nextBlockOnDiskSize, null); return hFileBlock; }
ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size); HFileContext meta = new HFileContextBuilder().build(); HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 52, -1, meta); HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, -1, -1, meta);
ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size); HFileContext meta = new HFileContextBuilder().build(); HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 52, -1, meta); HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, -1, -1, meta);
/** * Retrieves the decompressed/decrypted view of this block. An encoded block remains in its * encoded structure. Internal structures are shared between instances where applicable. */ HFileBlock unpack(HFileContext fileContext, FSReader reader) throws IOException { if (!fileContext.isCompressedOrEncrypted()) { // TODO: cannot use our own fileContext here because HFileBlock(ByteBuffer, boolean), // which is used for block serialization to L2 cache, does not preserve encoding and // encryption details. return this; } HFileBlock unpacked = new HFileBlock(this); unpacked.allocateBuffer(); // allocates space for the decompressed block HFileBlockDecodingContext ctx = blockType == BlockType.ENCODED_DATA ? reader.getBlockDecodingContext() : reader.getDefaultBlockDecodingContext(); ByteBuff dup = this.buf.duplicate(); dup.position(this.headerSize()); dup = dup.slice(); ctx.prepareDecoding(unpacked.getOnDiskSizeWithoutHeader(), unpacked.getUncompressedSizeWithoutHeader(), unpacked.getBufferWithoutHeader(), dup); return unpacked; }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
.withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .withChecksumType(ChecksumType.NULL).build(); HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 0, -1, meta); long byteBufferExpectedSize = ClassSize.align(ClassSize.estimateBase(
.withIncludesTags(fileContext.isIncludesTags()) .build(); return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), getUncompressedSizeWithoutHeader(), prevOffset, cacheConf.shouldCacheCompressed(blockType.getCategory())?
public HFileBlock deserialize(ByteBuffer buf) throws IOException{ ByteBuffer newByteBuffer = ByteBuffer.allocate(buf.limit() - HFileBlock.EXTRA_SERIALIZATION_SPACE); buf.limit(buf.limit() - HFileBlock.EXTRA_SERIALIZATION_SPACE).rewind(); newByteBuffer.put(buf); HFileBlock ourBuffer = new HFileBlock(newByteBuffer, MINOR_VERSION_NO_CHECKSUM); buf.position(buf.limit()); buf.limit(buf.limit() + HFileBlock.EXTRA_SERIALIZATION_SPACE); ourBuffer.offset = buf.getLong(); ourBuffer.nextBlockOnDiskSizeWithHeader = buf.getInt(); return ourBuffer; } };
/** * Creates a new HFileBlock. Checksums have already been validated, so * the byte buffer passed into the constructor of this newly created * block does not have checksum data even though the header minor * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a * 0 value in bytesPerChecksum. */ public HFileBlock getBlockForCaching() { return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), getUncompressedSizeWithoutHeader(), prevOffset, getUncompressedBufferWithHeader(), DONT_FILL_HEADER, startOffset, includesMemstoreTS, this.minorVersion, 0, ChecksumType.NULL.getCode(), // no checksums in cached data onDiskBytesWithHeader.length + onDiskChecksum.length); } }
@Test public void testSerializeWithoutNextBlockMetadata() { int size = 100; int length = HConstants.HFILEBLOCK_HEADER_SIZE + size; byte[] byteArr = new byte[length]; ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size); HFileContext meta = new HFileContextBuilder().build(); HFileBlock blockWithNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 52, -1, meta); HFileBlock blockWithoutNextBlockMetadata = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, -1, -1, meta); ByteBuffer buff1 = ByteBuffer.allocate(length); ByteBuffer buff2 = ByteBuffer.allocate(length); blockWithNextBlockMetadata.serialize(buff1, true); blockWithoutNextBlockMetadata.serialize(buff2, true); assertNotEquals(buff1, buff2); buff1.clear(); buff2.clear(); blockWithNextBlockMetadata.serialize(buff1, false); blockWithoutNextBlockMetadata.serialize(buff2, false); assertEquals(buff1, buff2); } }
private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) { ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE); buf.position(HConstants.HFILEBLOCK_HEADER_SIZE); keyValues.rewind(); buf.put(keyValues); HFileContext meta = new HFileContextBuilder() .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTag) .withHBaseCheckSum(true) .withCompression(Algorithm.NONE) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, meta); return b; }
private HFileBlock encodeDataBlock(HFileBlock block, DataBlockEncoding algo, boolean includesMemstoreTS) { ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer( block.getBufferWithoutHeader(), algo, includesMemstoreTS, block.getDummyHeaderForVersion()); int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize(); HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA, block.getOnDiskSizeWithoutHeader(), sizeWithoutHeader, block.getPrevBlockOffset(), compressedBuffer, HFileBlock.FILL_HEADER, block.getOffset(), includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); block.passSchemaMetricsTo(encodedBlock); return encodedBlock; }
private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException { int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM; // Create some KVs and create the block with old-style header. List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags); ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + headerSize); buf.position(headerSize); keyValues.rewind(); buf.put(keyValues); HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTags) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, hfileContext); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags); assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }