@Override public DataBlockEncoding getDataBlockEncoding() { return getStringOrDefault(DATA_BLOCK_ENCODING_BYTES, n -> DataBlockEncoding.valueOf(n.toUpperCase()), DataBlockEncoding.NONE); }
/** * @return name converted to bytes. */ public byte[] getNameInBytes() { return Bytes.toBytes(toString()); }
/** * Find and create data block encoder for given id; * @param encoderId id of data block encoder. * @return Newly created data block encoder. */ public static DataBlockEncoder getDataBlockEncoderById(short encoderId) { return getEncodingById(encoderId).getEncoder(); }
/** * Find and return the name of data block encoder for the given id. * @param encoderId id of data block encoder * @return name, same as used in options in column family */ public static String getNameFromId(short encoderId) { return getEncodingById(encoderId).toString(); }
@Test public void testGetDataBlockEncoder() throws Exception { for (DataBlockEncoding algo : DataBlockEncoding.values()) { DataBlockEncoder encoder = DataBlockEncoding.getDataBlockEncoderById(algo.getId()); if (algo.getId() != 0) { assertTrue(DataBlockEncoding.isCorrectEncoder(encoder, algo.getId())); } } try { DataBlockEncoding.getDataBlockEncoderById((short) -1); fail("Illegal encoderId, should get IllegalArgumentException."); } catch (IllegalArgumentException ie) { } try { DataBlockEncoding.getDataBlockEncoderById(Byte.MAX_VALUE); // fail because idArray[Byte.MAX_VALUE] = null fail("Illegal encoderId, should get IllegalArgumentException."); } catch (IllegalArgumentException ie) { } }
/** * Test whether the decompression of first key is implemented correctly. * @throws IOException */ @Test public void testFirstKeyInBlockOnSample() throws IOException { List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue; } DataBlockEncoder encoder = encoding.getEncoder(); ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv, getEncodingContext(Compression.Algorithm.NONE, encoding), this.useOffheapData); Cell key = encoder.getFirstKeyCellInBlock(new SingleByteBuff(encodedBuffer)); KeyValue firstKv = sampleKv.get(0); if (0 != PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, key, firstKv)) { int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(key, firstKv, false, true); fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix)); } } }
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { return this.encoding.getEncoder().encode(cell, encodingCtx, out); }
private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in, DataBlockEncoding algo, boolean includesMemstoreTS, byte[] dummyHeader) { ByteArrayOutputStream encodedStream = new ByteArrayOutputStream(); DataOutputStream dataOut = new DataOutputStream(encodedStream); DataBlockEncoder encoder = algo.getEncoder(); try { encodedStream.write(dummyHeader); algo.writeIdInBytes(dataOut); encoder.compressKeyValues(dataOut, in, includesMemstoreTS); } catch (IOException e) { throw new RuntimeException(String.format("Bug in data block encoder " + "'%s', it probably requested too much data", algo.toString()), e); } return ByteBuffer.wrap(encodedStream.toByteArray()); }
private static final List<DataBlockEncoding> createEncodingsToIterate() { List<DataBlockEncoding> encodings = new ArrayList<>(Arrays.asList(DataBlockEncoding.values())); encodings.add(DataBlockEncoding.NONE); return Collections.unmodifiableList(encodings); }
for (DataBlockEncoding encoding : DataBlockEncoding.values()) { LOG.info("testDataBlockEncoding: Compression algorithm={}, pread={}, dataBlockEncoder={}", algo.toString(), pread, encoding); Path path = new Path(TEST_UTIL.getDataTestDir(), "blocks_v2_" + algo + "_" + encoding.toString()); FSDataOutputStream os = fs.create(path); HFileDataBlockEncoder dataBlockEncoder = (encoding != DataBlockEncoding.NONE) ? assertEquals( "Unexpected second byte with " + buildMessageDetails(algo, encoding, pread), Long.toHexString(encoding.getId()), Long.toHexString(actualBuffer.get(1))); actualBuffer.position(2); actualBuffer = actualBuffer.slice();
/** * Check if given encoder has this id. * @param encoder encoder which id will be checked * @param encoderId id which we except * @return true if id is right for given encoder, false otherwise * @exception IllegalArgumentException * thrown when there is no matching data block encoder */ public static boolean isCorrectEncoder(DataBlockEncoder encoder, short encoderId) { DataBlockEncoding algorithm = getEncodingById(encoderId); String encoderCls = encoder.getClass().getName(); return encoderCls.equals(algorithm.encoderCls); }
@Override public String toString() { return encoding.name(); } }
short id = possibleIds[RandomUtils.nextInt(0, possibleIds.length)].getId(); LOG.info("Altering encoding of column family: " + columnDesc + " to: " + id + " in table: " + tableName); .setDataBlockEncoding(DataBlockEncoding.getEncodingById(id)) .build(); TableDescriptor td = TableDescriptorBuilder.newBuilder(selected) ColumnFamilyDescriptor freshColumnDesc = freshTableDesc.getColumnFamily(columnDesc.getName()); Assert.assertEquals("Encoding of column family: " + columnDesc + " was not altered", freshColumnDesc.getDataBlockEncoding().getId(), id); Assert.assertTrue( "After alter encoding of column family, Table: " + tableName + " is not disabled",
if (!(encoding.equals(DataBlockEncoding.ROW_INDEX_V1) && noTags)) { tagsLen = ((decompressedData.get() & 0xff) << 8) ^ (decompressedData.get() & 0xff); ByteBufferUtils.skip(decompressedData, tagsLen);
private void seekToTheKey(KeyValue expected, List<KeyValue> kvs, Cell toSeek) throws IOException { // create all seekers List<DataBlockEncoder.EncodedSeeker> encodedSeekers = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue; } DataBlockEncoder encoder = encoding.getEncoder(); HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(false).withIncludesTags(false) .withCompression(Compression.Algorithm.NONE).build(); HFileBlockEncodingContext encodingContext = encoder.newDataBlockEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, meta); ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs, encodingContext, this.useOffheapData); DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker); } // test it! // try a few random seeks checkSeekingConsistency(encodedSeekers, toSeek, expected); }
/** * Find and return the name of data block encoder for the given id. * @param encoderId id of data block encoder * @return name, same as used in options in column family */ public static String getNameFromId(short encoderId) { return getEncodingById(encoderId).toString(); }
@Override public void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { if (this.encoding != null && this.encoding != DataBlockEncoding.NONE) { this.encoding.getEncoder().startBlockEncoding(encodingCtx, out); } }
@Parameters public static Collection<Object[]> parameters() { List<Object[]> paramList = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { paramList.add(new Object[] { encoding }); } return paramList; } static boolean switchKVs = false;
DataBlockEncoding getDataBlockEncoding() { if (blockType == BlockType.ENCODED_DATA) { return DataBlockEncoding.getEncodingById(getDataBlockEncodingId()); } return DataBlockEncoding.NONE; }
/** * Set data block encoding algorithm used in block cache. * * @param type What kind of data block encoding will be used. * @return this (for chained invocation) */ public ModifyableColumnFamilyDescriptor setDataBlockEncoding(DataBlockEncoding type) { return setValue(DATA_BLOCK_ENCODING_BYTES, type == null ? DataBlockEncoding.NONE.name() : type.name()); }