public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path, FSDataOutputStream outputStream, CellComparator comparator, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } this.comparator = comparator != null ? comparator : CellComparator.getInstance(); closeOutputStream = path != null; this.cacheConf = cacheConf; float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f); this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio); finishInit(conf); if (LOG.isTraceEnabled()) { LOG.trace("Writer" + (path != null ? " for " + path : "") + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + fileContext); } }
/** * At a block boundary, write all the inline blocks and opens new block. * * @throws IOException */ protected void checkBlockBoundary() throws IOException { //for encoder like prefixTree, encoded size is not available, so we have to compare both encoded size //and unencoded size to blocksize limit. if (blockWriter.encodedBlockSizeWritten() >= encodedBlockSizeLimit || blockWriter.blockSizeWritten() >= hFileContext.getBlocksize()) { finishBlock(); writeInlineBlocks(false); newBlock(); } }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
.withBlockSize(fileContext.getBlocksize()) .withBytesPerCheckSum(0)
/** * At a block boundary, write all the inline blocks and opens new block. * * @throws IOException */ protected void checkBlockBoundary() throws IOException { if (fsBlockWriter.blockSizeWritten() < hFileContext.getBlocksize()) return; finishBlock(); writeInlineBlocks(false); newBlock(); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
/** * Creates a new HFileBlock. Checksums have already been validated, so * the byte buffer passed into the constructor of this newly created * block does not have checksum data even though the header minor * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a * 0 value in bytesPerChecksum. */ public HFileBlock getBlockForCaching(CacheConfig cacheConf) { HFileContext newContext = new HFileContextBuilder() .withBlockSize(fileContext.getBlocksize()) .withBytesPerCheckSum(0) .withChecksumType(ChecksumType.NULL) // no checksums in cached data .withCompression(fileContext.getCompression()) .withDataBlockEncoding(fileContext.getDataBlockEncoding()) .withHBaseCheckSum(fileContext.isUseHBaseChecksum()) .withCompressTags(fileContext.isCompressTags()) .withIncludesMvcc(fileContext.isIncludesMvcc()) .withIncludesTags(fileContext.isIncludesTags()) .build(); return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), getUncompressedSizeWithoutHeader(), prevOffset, cacheConf.shouldCacheCompressed(blockType.getCategory()) ? getOnDiskBufferWithHeader() : getUncompressedBufferWithHeader(), FILL_HEADER, startOffset, onDiskBytesWithHeader.length + onDiskChecksum.length, newContext); } }