@Override public String toString() { return this.getColumnFamilyName(); }
/** * This method should only be called from Region. It is assumed that the ranges of values in the * HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this) * * @param srcPathStr * @param seqNum sequence Id associated with the HFile */ public Pair<Path, Path> preBulkLoadHFile(String srcPathStr, long seqNum) throws IOException { Path srcPath = new Path(srcPathStr); return fs.bulkLoadStoreFile(getColumnFamilyName(), srcPath, seqNum); }
/** * Replaces the store files that the store has with the given files. Mainly used by secondary * region replicas to keep up to date with the primary region files. * @throws IOException */ public void refreshStoreFiles(Collection<String> newFiles) throws IOException { List<StoreFileInfo> storeFiles = new ArrayList<>(newFiles.size()); for (String file : newFiles) { storeFiles.add(fs.getStoreFileInfo(getColumnFamilyName(), file)); } refreshStoreFilesInternal(storeFiles); }
/** * Creates an unsorted list of StoreFile loaded in parallel * from the given directory. * @throws IOException */ private List<HStoreFile> loadStoreFiles() throws IOException { Collection<StoreFileInfo> files = fs.getStoreFiles(getColumnFamilyName()); return openStoreFiles(files); }
@Override public void refreshStoreFiles() throws IOException { Collection<StoreFileInfo> newFiles = fs.getStoreFiles(getColumnFamilyName()); refreshStoreFilesInternal(newFiles); }
/** * Adds a value to the memstore */ public void add(final Cell cell, MemStoreSizing memstoreSizing) { lock.readLock().lock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace(this.getTableName() + "tableName={}, encodedName={}, columnFamilyName={} is " + "too busy!", this.getRegionInfo().getEncodedName(), this .getColumnFamilyName()); } this.memstore.add(cell, memstoreSizing); } finally { lock.readLock().unlock(); currentParallelPutCount.decrementAndGet(); } }
/** * Adds the specified value to the memstore */ public void add(final Iterable<Cell> cells, MemStoreSizing memstoreSizing) { lock.readLock().lock(); try { if (this.currentParallelPutCount.getAndIncrement() > this.parallelPutCountPrintThreshold) { LOG.trace(this.getTableName() + "tableName={}, encodedName={}, columnFamilyName={} is " + "too busy!", this.getRegionInfo().getEncodedName(), this .getColumnFamilyName()); } memstore.add(cells, memstoreSizing); } finally { lock.readLock().unlock(); currentParallelPutCount.decrementAndGet(); } }
HStoreFile moveFileIntoPlace(Path newFile) throws IOException { validateStoreFile(newFile); // Move the file into the right spot Path destPath = fs.commitStoreFile(getColumnFamilyName(), newFile); return createStoreFileAndReader(destPath); }
public DefaultMobStoreFlusher(Configuration conf, HStore store) throws IOException { super(conf, store); if (!(store instanceof HMobStore)) { throw new IllegalArgumentException("The store " + store + " is not a HMobStore"); } mobCellValueSizeThreshold = store.getColumnFamilyDescriptor().getMobThreshold(); this.targetPath = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName()); if (!this.store.getFileSystem().exists(targetPath)) { this.store.getFileSystem().mkdirs(targetPath); } this.mobStore = (HMobStore) store; }
/** * Be careful, this method will drop all data in the memstore of this region. * Currently, this method is used to drop memstore to prevent memory leak * when replaying recovered.edits while opening region. */ public MemStoreSize dropMemStoreContents() throws IOException { MemStoreSizing totalFreedSize = new NonThreadSafeMemStoreSizing(); this.updatesLock.writeLock().lock(); try { for (HStore s : stores.values()) { MemStoreSize memStoreSize = doDropStoreMemStoreContentsForSeqId(s, HConstants.NO_SEQNUM); LOG.info("Drop memstore for Store " + s.getColumnFamilyName() + " in region " + this.getRegionInfo().getRegionNameAsString() + " , dropped memstoresize: [" + memStoreSize + " }"); totalFreedSize.incMemStoreSize(memStoreSize); } return totalFreedSize.getMemStoreSize(); } finally { this.updatesLock.writeLock().unlock(); } }
public Path bulkLoadHFile(byte[] family, String srcPathStr, Path dstPath) throws IOException { Path srcPath = new Path(srcPathStr); try { fs.commitStoreFile(srcPath, dstPath); } finally { if (this.getCoprocessorHost() != null) { this.getCoprocessorHost().postCommitStoreFile(family, srcPath, dstPath); } } LOG.info("Loaded HFile " + srcPath + " into store '" + getColumnFamilyName() + "' as " + dstPath + " - updating store file list."); HStoreFile sf = createStoreFileAndReader(dstPath); bulkLoadHFile(sf); LOG.info("Successfully loaded store file {} into store {} (new location: {})", srcPath, this, dstPath); return dstPath; }
/** * Utility method broken out of internalPrepareFlushCache so that method is smaller. */ private void logFatLineOnFlush(Collection<HStore> storesToFlush, long sequenceId) { if (!LOG.isInfoEnabled()) { return; } // Log a fat line detailing what is being flushed. StringBuilder perCfExtras = null; if (!isAllFamilies(storesToFlush)) { perCfExtras = new StringBuilder(); for (HStore store: storesToFlush) { MemStoreSize mss = store.getFlushableSize(); perCfExtras.append("; ").append(store.getColumnFamilyName()); perCfExtras.append("={dataSize=") .append(StringUtils.byteDesc(mss.getDataSize())); perCfExtras.append(", heapSize=") .append(StringUtils.byteDesc(mss.getHeapSize())); perCfExtras.append(", offHeapSize=") .append(StringUtils.byteDesc(mss.getOffHeapSize())); perCfExtras.append("}"); } } MemStoreSize mss = this.memStoreSizing.getMemStoreSize(); LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() + " column families," + " dataSize=" + StringUtils.byteDesc(mss.getDataSize()) + " heapSize=" + StringUtils.byteDesc(mss.getHeapSize()) + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + sequenceId)); }
private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); // Append the new storefile into the list this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); } finally { // We need the lock, as long as we are updating the storeFiles // or changing the memstore. Let us release it before calling // notifyChangeReadersObservers. See HBASE-4485 for a possible // deadlock scenario that could have happened if continue to hold // the lock. this.lock.writeLock().unlock(); } LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName()); if (LOG.isTraceEnabled()) { String traceMessage = "BULK LOAD time,size,store size,store files [" + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } }
public CompactingMemStore(Configuration conf, CellComparator c, HStore store, RegionServicesForStores regionServices, MemoryCompactionPolicy compactionPolicy) throws IOException { super(conf, c, regionServices); this.store = store; this.regionServices = regionServices; this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = createMemStoreCompactor(compactionPolicy); if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) { // if user requested to work with MSLABs (whether on- or off-heap), then the // immutable segments are going to use CellChunkMap as their index indexType = IndexType.CHUNK_MAP; } else { indexType = IndexType.ARRAY_MAP; } // initialization of the flush size should happen after initialization of the index type // so do not transfer the following method initInmemoryFlushSize(conf); LOG.info("Store={}, in-memory flush size threshold={}, immutable segments index type={}, " + "compactor={}", this.store.getColumnFamilyName(), StringUtils.byteDesc(this.inmemoryFlushSize), this.indexType, (this.compactor == null? "NULL": this.compactor.toString())); }
/** * @param path The pathname of the tmp file into which the store was flushed * @param logCacheFlushId * @param status * @return store file created. * @throws IOException */ private HStoreFile commitFile(Path path, long logCacheFlushId, MonitoredTask status) throws IOException { // Write-out finished successfully, move into the right spot Path dstPath = fs.commitStoreFile(getColumnFamilyName(), path); status.setStatus("Flushing " + this + ": reopening flushed file"); HStoreFile sf = createStoreFileAndReader(dstPath); StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); if (LOG.isInfoEnabled()) { LOG.info("Added " + sf + ", entries=" + r.getEntries() + ", sequenceid=" + logCacheFlushId + ", filesize=" + TraditionalBinaryPrefix.long2String(r.length(), "", 1)); } return sf; }
private void archiveStoreFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStorefiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } store.getRegionFileSystem().removeStoreFiles(store.getColumnFamilyName(), Lists.newArrayList(sf)); }
@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); for (HStore store : region.getStores()) { // If any of the stores is unable to split (eg they contain reference files) // then don't split if (!store.canSplit()) { return false; } // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + StringUtils.humanSize(size) + ", sizeToCheck=" + StringUtils.humanSize(sizeToCheck) + ", regionsWithCommonTable=" + tableRegionsCount); foundABigStore = true; } } return foundABigStore || force; }
@Test public void testSkipRecoveredEditsReplayAllIgnored() throws Exception { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); Path regiondir = region.getRegionFileSystem().getRegionDir(); FileSystem fs = region.getRegionFileSystem().getFileSystem(); Path recoveredEditsDir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); for (int i = 1000; i < 1050; i += 10) { Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", i)); FSDataOutputStream dos = fs.create(recoveredEdits); dos.writeInt(i); dos.close(); } long minSeqId = 2000; Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", minSeqId - 1)); FSDataOutputStream dos = fs.create(recoveredEdits); dos.close(); Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR); for (HStore store : region.getStores()) { maxSeqIdInStores.put(Bytes.toBytes(store.getColumnFamilyName()), minSeqId); } long seqId = region.replayRecoveredEditsIfAny(maxSeqIdInStores, null, null); assertEquals(minSeqId, seqId); }
protected boolean shouldFlush(HStore store) { if (store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize() > this.flushSizeLowerBound) { LOG.debug("Flush {} of {}; " + "heap memstoreSize={} +" + "off heap memstoreSize={} > memstore lowerBound={}" , store.getColumnFamilyName() , region.getRegionInfo().getEncodedName() , store.getMemStoreSize().getHeapSize() , store.getMemStoreSize().getOffHeapSize() , this.flushSizeLowerBound ); return true; } return false; } }
@Override public Object run() throws Exception { // Make sure it worked (above is sensitive to caching details in hadoop core) FileSystem fs = FileSystem.get(conf); assertEquals(FaultyFileSystem.class, fs.getClass()); // Initialize region init(name.getMethodName(), conf); LOG.info("Adding some data"); store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); LOG.info("Before flush, we should have no files"); Collection<StoreFileInfo> files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); assertEquals(0, files != null ? files.size() : 0); //flush try { LOG.info("Flushing"); flush(1); fail("Didn't bubble up IOE!"); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains("Fault injected")); } LOG.info("After failed flush, we should still have no files!"); files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); assertEquals(0, files != null ? files.size() : 0); store.getHRegion().getWAL().close(); return null; } });