private ThreadPoolExecutor getPool() { return getRegionServices().getInMemoryCompactionPool(); }
private void initInmemoryFlushSize(Configuration conf) { double factor = 0; long memstoreFlushSize = getRegionServices().getMemStoreFlushSize(); int numStores = getRegionServices().getNumStores(); if (numStores <= 1) { // Family number might also be zero in some of our unit test case numStores = 1; } factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.0); if(factor != 0.0) { // multiply by a factor (the same factor for all index types) inmemoryFlushSize = (long) (factor * memstoreFlushSize) / numStores; } else { inmemoryFlushSize = IN_MEMORY_FLUSH_MULTIPLIER * conf.getLong(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT); inmemoryFlushSize -= ChunkCreator.SIZEOF_CHUNK_HEADER; } }
@Override public void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater) { long minSequenceId = pipeline.getMinSequenceId(); if(minSequenceId != Long.MAX_VALUE) { byte[] encodedRegionName = getRegionServices().getRegionInfo().getEncodedNameAsBytes(); byte[] familyName = getFamilyNameInBytes(); WAL WAL = getRegionServices().getWAL(); if (WAL != null) { WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater); } } }
void inMemoryCompaction() { // setting the inMemoryCompactionInProgress flag again for the case this method is invoked // directly (only in tests) in the common path setting from true to true is idempotent inMemoryCompactionInProgress.set(true); // Used by tests if (!allowCompaction.get()) { return; } try { // Speculative compaction execution, may be interrupted if flush is forced while // compaction is in progress if(!compactor.start()) { setInMemoryCompactionCompleted(); } } catch (IOException e) { LOG.warn("Unable to run in-memory compaction on {}/{}; exception={}", getRegionServices().getRegionInfo().getEncodedName(), getFamilyName(), e); } }
public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy) throws IOException { super(conf, c, regionServices); this.store = store; this.regionServices = regionServices; this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = createMemStoreCompactor(compactionPolicy); if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) { // if user requested to work with MSLABs (whether on- or off-heap), then the // immutable segments are going to use CellChunkMap as their index indexType = IndexType.CHUNK_MAP; } else { indexType = IndexType.ARRAY_MAP; } // initialization of the flush size should happen after initialization of the index type // so do not transfer the following method initInmemoryFlushSize(conf); LOG.info("Store={}, in-memory flush size threshold={}, immutable segments index type={}, " + "compactor={}", this.store.getColumnFamilyName(), StringUtils.byteDesc(this.inmemoryFlushSize), this.indexType, (this.compactor == null? "NULL": this.compactor.toString())); }
} else { LOG.debug("FLUSHING TO DISK {}, store={}", getRegionServices().getRegionInfo().getEncodedName(), getFamilyName()); stopCompaction();