public BufferedEncodedSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx) { super(comparator, decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } } current = createSeekerState(); // always valid previous = createSeekerState(); // may not be valid }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
protected void finishFileInfo() throws IOException { if (lastCell != null) { // Make a copy. The copy is stuffed into our fileinfo map. Needs a clean // byte buffer. Won't take a tuple. byte [] lastKey = PrivateCellUtil.getCellKeySerializedAsKeyValueKey(this.lastCell); fileInfo.append(FileInfo.LASTKEY, lastKey, false); } // Average key length. int avgKeyLen = entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount); fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false); fileInfo.append(FileInfo.CREATE_TIME_TS, Bytes.toBytes(hFileContext.getFileCreateTime()), false); // Average value length. int avgValueLen = entryCount == 0 ? 0 : (int) (totalValueLength / entryCount); fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false); if (hFileContext.isIncludesTags()) { // When tags are not being written in this file, MAX_TAGS_LEN is excluded // from the FileInfo fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false); boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE) && hFileContext.isCompressTags(); fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false); } }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); if (encodingCtx.getHFileContext().isIncludesTags() && encodingCtx.getHFileContext().isCompressTags()) { if (encodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // encoding. encodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } StreamUtils.writeInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding() blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState()); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
.withDataBlockEncoding(fileContext.getDataBlockEncoding()) .withHBaseCheckSum(fileContext.isUseHBaseChecksum()) .withCompressTags(fileContext.isCompressTags()) .withIncludesMvcc(fileContext.isIncludesMvcc()) .withIncludesTags(fileContext.isIncludesTags())
public BufferedEncodedSeeker(KVComparator comparator, HFileBlockDecodingContext decodingCtx) { this.comparator = comparator; this.samePrefixComparator = comparator; this.decodingCtx = decodingCtx; if (decodingCtx.getHFileContext().isCompressTags()) { try { tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } } }
public BufferedEncodedSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx) { super(comparator, decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } } current = createSeekerState(); // always valid previous = createSeekerState(); // may not be valid }
public BufferedEncodedSeeker(CellComparator comparator, HFileBlockDecodingContext decodingCtx) { super(comparator, decodingCtx); if (decodingCtx.getHFileContext().isCompressTags()) { try { tagCompressionContext = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); } catch (Exception e) { throw new RuntimeException("Failed to initialize TagCompressionContext", e); } } current = createSeekerState(); // always valid previous = createSeekerState(); // may not be valid }
protected void finishFileInfo() throws IOException { super.finishFileInfo(); if (hFileContext.getDataBlockEncoding() == DataBlockEncoding.PREFIX_TREE) { // In case of Prefix Tree encoding, we always write tags information into HFiles even if all // KVs are having no tags. fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false); } else if (hFileContext.isIncludesTags()) { // When tags are not being written in this file, MAX_TAGS_LEN is excluded // from the FileInfo fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false); boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE) && hFileContext.isCompressTags(); fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false); } }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); if (encodingCtx.getHFileContext().isIncludesTags() && encodingCtx.getHFileContext().isCompressTags()) { if (encodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // encoding. encodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } StreamUtils.writeInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding() blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState()); }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); if (encodingCtx.getHFileContext().isIncludesTags() && encodingCtx.getHFileContext().isCompressTags()) { if (encodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // encoding. encodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } ByteBufferUtils.putInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding() blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState()); }
@Override public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out) throws IOException { if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) { throw new IOException (this.getClass().getName() + " only accepts " + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context."); } HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx; encodingCtx.prepareEncoding(out); if (encodingCtx.getHFileContext().isIncludesTags() && encodingCtx.getHFileContext().isCompressTags()) { if (encodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // encoding. encodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); encodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } StreamUtils.writeInt(out, 0); // DUMMY length. This will be updated in endBlockEncoding() blkEncodingCtx.setEncodingState(new BufferedDataBlockEncodingState()); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
/** * Use this constructor if you want to change a few settings only in another context. */ public HFileContextBuilder(final HFileContext hfc) { this.usesHBaseChecksum = hfc.isUseHBaseChecksum(); this.includesMvcc = hfc.isIncludesMvcc(); this.includesTags = hfc.isIncludesTags(); this.compression = hfc.getCompression(); this.compressTags = hfc.isCompressTags(); this.checksumType = hfc.getChecksumType(); this.bytesPerChecksum = hfc.getBytesPerChecksum(); this.blocksize = hfc.getBlocksize(); this.encoding = hfc.getDataBlockEncoding(); this.cryptoContext = hfc.getEncryptionContext(); this.fileCreateTime = hfc.getFileCreateTime(); this.hfileName = hfc.getHFileName(); }
/** * Creates a new HFileBlock. Checksums have already been validated, so * the byte buffer passed into the constructor of this newly created * block does not have checksum data even though the header minor * version is MINOR_VERSION_WITH_CHECKSUM. This is indicated by setting a * 0 value in bytesPerChecksum. */ public HFileBlock getBlockForCaching(CacheConfig cacheConf) { HFileContext newContext = new HFileContextBuilder() .withBlockSize(fileContext.getBlocksize()) .withBytesPerCheckSum(0) .withChecksumType(ChecksumType.NULL) // no checksums in cached data .withCompression(fileContext.getCompression()) .withDataBlockEncoding(fileContext.getDataBlockEncoding()) .withHBaseCheckSum(fileContext.isUseHBaseChecksum()) .withCompressTags(fileContext.isCompressTags()) .withIncludesMvcc(fileContext.isIncludesMvcc()) .withIncludesTags(fileContext.isIncludesTags()) .build(); return new HFileBlock(blockType, getOnDiskSizeWithoutHeader(), getUncompressedSizeWithoutHeader(), prevOffset, cacheConf.shouldCacheCompressed(blockType.getCategory()) ? getOnDiskBufferWithHeader() : getUncompressedBufferWithHeader(), FILL_HEADER, startOffset, onDiskBytesWithHeader.length + onDiskChecksum.length, newContext); } }