int getBytesPerChecksum() { return this.fileContext.getBytesPerChecksum(); }
/** * Calculate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ int totalChecksumBytes() { // If the hfile block has minorVersion 0, then there are no checksum // data to validate. Similarly, a zero value in this.bytesPerChecksum // indicates that cached blocks do not have checksum data because // checksums were already validated when the block was read from disk. if (!fileContext.isUseHBaseChecksum() || this.fileContext.getBytesPerChecksum() == 0) { return 0; } return (int) ChecksumUtil.numBytes(onDiskDataSizeWithHeader, this.fileContext.getBytesPerChecksum()); }
/** * @param dataBlockEncoder data block encoding algorithm to use */ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) { if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + fileContext.getBytesPerChecksum()); } this.dataBlockEncoder = dataBlockEncoder != null? dataBlockEncoder: NoOpDataBlockEncoder.INSTANCE; this.dataBlockEncodingCtx = this.dataBlockEncoder. newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: This should be lazily instantiated since we usually do NOT need this default encoder this.defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: Set BAOS initial size. Use fileContext.getBlocksize() and add for header/checksum baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) { prevOffsetByType[i] = UNSET; } // TODO: Why fileContext saved away when we have dataBlockEncoder and/or // defaultDataBlockEncoder? this.fileContext = fileContext; }
/** * Clones the header followed by the uncompressed data, even if using * compression. This is needed for storing uncompressed blocks in the block * cache. Can be called in the "writing" state or the "block ready" state. * Returns only the header and data, does not include checksum data. * * @return Returns a copy of uncompressed block bytes for caching on write */ @VisibleForTesting ByteBuffer cloneUncompressedBufferWithHeader() { expectState(State.BLOCK_READY); byte[] uncompressedBlockBytesWithHeader = baosInMemory.toByteArray(); int numBytes = (int) ChecksumUtil.numBytes( onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); putHeader(uncompressedBlockBytesWithHeader, 0, onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(), onDiskBlockBytesWithHeader.size()); return ByteBuffer.wrap(uncompressedBlockBytesWithHeader); }
/** * Put the header into the given byte array at the given offset. * @param onDiskSize size of the block on disk header + data + checksum * @param uncompressedSize size of the block after decompression (but * before optional data block decoding) including header * @param onDiskDataSize size of the block on disk with header * and data but not including the checksums */ private void putHeader(byte[] dest, int offset, int onDiskSize, int uncompressedSize, int onDiskDataSize) { offset = blockType.put(dest, offset); offset = Bytes.putInt(dest, offset, onDiskSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putInt(dest, offset, uncompressedSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putLong(dest, offset, prevOffset); offset = Bytes.putByte(dest, offset, fileContext.getChecksumType().getCode()); offset = Bytes.putInt(dest, offset, fileContext.getBytesPerChecksum()); Bytes.putInt(dest, offset, onDiskDataSize); }
/** * Rewinds {@code buf} and writes first 4 header fields. {@code buf} position * is modified as side-effect. */ private void overwriteHeader() { buf.rewind(); blockType.write(buf); buf.putInt(onDiskSizeWithoutHeader); buf.putInt(uncompressedSizeWithoutHeader); buf.putLong(prevBlockOffset); if (this.fileContext.isUseHBaseChecksum()) { buf.put(fileContext.getChecksumType().getCode()); buf.putInt(fileContext.getBytesPerChecksum()); buf.putInt(onDiskDataSizeWithHeader); } }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
fileContext.getBytesPerChecksum()); onDiskChecksum, 0, fileContext.getChecksumType(), fileContext.getBytesPerChecksum());
if (this.fileContext.isUseHBaseChecksum()) { sanityCheckAssertion(dup.get(), this.fileContext.getChecksumType().getCode(), "checksumType"); sanityCheckAssertion(dup.getInt(), this.fileContext.getBytesPerChecksum(), "bytesPerChecksum"); sanityCheckAssertion(dup.getInt(), onDiskDataSizeWithHeader, "onDiskDataSizeWithHeader");
int getBytesPerChecksum() { return this.fileContext.getBytesPerChecksum(); }
/** * Calcuate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ int totalChecksumBytes() { // If the hfile block has minorVersion 0, then there are no checksum // data to validate. Similarly, a zero value in this.bytesPerChecksum // indicates that cached blocks do not have checksum data because // checksums were already validated when the block was read from disk. if (!fileContext.isUseHBaseChecksum() || this.fileContext.getBytesPerChecksum() == 0) { return 0; } return (int) ChecksumUtil.numBytes(onDiskDataSizeWithHeader, this.fileContext.getBytesPerChecksum()); }
/** * Rewinds {@code buf} and writes first 4 header fields. {@code buf} position * is modified as side-effect. */ private void overwriteHeader() { buf.rewind(); blockType.write(buf); buf.putInt(onDiskSizeWithoutHeader); buf.putInt(uncompressedSizeWithoutHeader); buf.putLong(prevBlockOffset); if (this.fileContext.isUseHBaseChecksum()) { buf.put(fileContext.getChecksumType().getCode()); buf.putInt(fileContext.getBytesPerChecksum()); buf.putInt(onDiskDataSizeWithHeader); } }
/** * @param dataBlockEncoder data block encoding algorithm to use */ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) { this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE; defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); dataBlockEncodingCtx = this.dataBlockEncoder .newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + fileContext.getBytesPerChecksum()); } baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) prevOffsetByType[i] = -1; this.fileContext = fileContext; }
if (this.fileContext.isUseHBaseChecksum()) { sanityCheckAssertion(buf.get(), this.fileContext.getChecksumType().getCode(), "checksumType"); sanityCheckAssertion(buf.getInt(), this.fileContext.getBytesPerChecksum(), "bytesPerChecksum"); sanityCheckAssertion(buf.getInt(), onDiskDataSizeWithHeader, "onDiskDataSizeWithHeader");
fileContext.getBytesPerChecksum()); ChecksumUtil.generateChecksums( onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length, onDiskChecksum, 0, fileContext.getChecksumType(), fileContext.getBytesPerChecksum());
/** * Put the header into the given byte array at the given offset. * @param onDiskSize size of the block on disk header + data + checksum * @param uncompressedSize size of the block after decompression (but * before optional data block decoding) including header * @param onDiskDataSize size of the block on disk with header * and data but not including the checksums */ private void putHeader(byte[] dest, int offset, int onDiskSize, int uncompressedSize, int onDiskDataSize) { offset = blockType.put(dest, offset); offset = Bytes.putInt(dest, offset, onDiskSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putInt(dest, offset, uncompressedSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putLong(dest, offset, prevOffset); offset = Bytes.putByte(dest, offset, fileContext.getChecksumType().getCode()); offset = Bytes.putInt(dest, offset, fileContext.getBytesPerChecksum()); Bytes.putInt(dest, offset, onDiskDataSize); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }