public static HFileDataBlockEncoder createFromFileInfo( FileInfo fileInfo) throws IOException { DataBlockEncoding encoding = DataBlockEncoding.NONE; byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING); if (dataBlockEncodingType != null) { String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); try { encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); } catch (IllegalArgumentException ex) { throw new IOException("Invalid data block encoding type in file info: " + dataBlockEncodingStr, ex); } } if (encoding == DataBlockEncoding.NONE) { return NoOpDataBlockEncoder.INSTANCE; } return new HFileDataBlockEncoderImpl(encoding); }
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path, FSDataOutputStream outputStream, CellComparator comparator, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } this.comparator = comparator != null ? comparator : CellComparator.getInstance(); closeOutputStream = path != null; this.cacheConf = cacheConf; float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f); this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio); finishInit(conf); if (LOG.isTraceEnabled()) { LOG.trace("Writer" + (path != null ? " for " + path : "") + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + fileContext); } }
/** * @return All possible data block encoding configurations */ @Parameters public static Collection<Object[]> getAllConfigurations() { List<Object[]> configurations = new ArrayList<>(); for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { for (boolean includesMemstoreTS : new boolean[] { false, true }) { HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); } } return configurations; } }
public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception { Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>(); for (HStore store : r.getStores()) { HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder(); replaceBlockCache.put(store, blockEncoder); final DataBlockEncoding inCache = DataBlockEncoding.PREFIX; final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache; ((HStore)store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk)); } majorCompaction(); // restore settings for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) { ((HStore)entry.getKey()).setDataBlockEncoderInTest(entry.getValue()); } }
this.dataBlockEncoder = new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
DataBlockEncoding.FAST_DIFF; HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl( dataBlockEncoderAlgo); cacheConf = new CacheConfig(conf);
FSDataOutputStream os = fs.create(path); HFileDataBlockEncoder dataBlockEncoder = (encoding != DataBlockEncoding.NONE) ? new HFileDataBlockEncoderImpl(encoding) : NoOpDataBlockEncoder.INSTANCE; HFileContext meta = new HFileContextBuilder() .withCompression(algo)
public AbstractHFileWriter(CacheConfig cacheConf, FSDataOutputStream outputStream, Path path, KVComparator comparator, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } this.comparator = comparator != null ? comparator : KeyValue.COMPARATOR; closeOutputStream = path != null; this.cacheConf = cacheConf; }
public static HFileDataBlockEncoder createFromFileInfo( FileInfo fileInfo) throws IOException { DataBlockEncoding encoding = DataBlockEncoding.NONE; byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING); if (dataBlockEncodingType != null) { String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); try { encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); } catch (IllegalArgumentException ex) { throw new IOException("Invalid data block encoding type in file info: " + dataBlockEncodingStr, ex); } } if (encoding == DataBlockEncoding.NONE) { return NoOpDataBlockEncoder.INSTANCE; } return new HFileDataBlockEncoderImpl(encoding); }
return new HFileDataBlockEncoderImpl(onDisk, inCache);
/** * @return All possible data block encoding configurations */ @Parameters public static Collection<Object[]> getAllConfigurations() { List<Object[]> configurations = new ArrayList<>(); for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { for (boolean includesMemstoreTS : new boolean[] { false, true }) { HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); } } return configurations; } }
} else { try { encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding .valueOf(dataBlockEncodingStr)); } catch (IllegalArgumentException ex) {
public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception { Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>(); for (HStore store : r.getStores()) { HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder(); replaceBlockCache.put(store, blockEncoder); final DataBlockEncoding inCache = DataBlockEncoding.PREFIX; final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache; ((HStore)store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk)); } majorCompaction(); // restore settings for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) { ((HStore)entry.getKey()).setDataBlockEncoderInTest(entry.getValue()); } }
new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(), family.getDataBlockEncoding());
new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
HalfStoreFileReader halfReader = null; StoreFile.Writer halfWriter = null; HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl( familyDescriptor.getDataBlockEncodingOnDisk(), familyDescriptor.getDataBlockEncoding());
DataBlockEncoding.FAST_DIFF; HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl( dataBlockEncoderAlgo); cacheConf = new CacheConfig(conf);
FSDataOutputStream os = fs.create(path); HFileDataBlockEncoder dataBlockEncoder = (encoding != DataBlockEncoding.NONE) ? new HFileDataBlockEncoderImpl(encoding) : NoOpDataBlockEncoder.INSTANCE; HFileContext meta = new HFileContextBuilder() .withCompression(algo)