private long getNumHFilesForRegion(HRegion region) { return region.getStores().stream().mapToLong((s) -> s.getNumHFiles()).sum(); }
@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; for (HStore store : region.getStores()) { // If any of the stores are unable to split (eg they contain reference files) // then don't split if ((!store.canSplit())) { return false; } // Mark if any store is big enough if (store.getSize() > desiredMaxFileSize) { foundABigStore = true; } } return foundABigStore || force; }
@Before public void setupMocks() { conf = HBaseConfiguration.create(); HRegionInfo hri = new HRegionInfo(TABLENAME); htd = new HTableDescriptor(TABLENAME); mockRegion = Mockito.mock(HRegion.class); Mockito.doReturn(htd).when(mockRegion).getTableDescriptor(); Mockito.doReturn(hri).when(mockRegion).getRegionInfo(); stores = new ArrayList<>(); Mockito.doReturn(stores).when(mockRegion).getStores(); }
@Override protected boolean shouldSplit() { float blockedReqRate = updateRate(); if (super.shouldSplit()) { return true; } if (EnvironmentEdgeManager.currentTime() < startTime + minAge) { return false; } for (HStore store: region.getStores()) { if (!store.canSplit()) { return false; } } if (blockedReqRate >= maxBlockedRequests) { if (LOG.isDebugEnabled()) { LOG.debug("Going to split region " + region.getRegionInfo().getRegionNameAsString() + " because it's too busy. Blocked Request rate: " + blockedReqRate); } return true; } return false; }
private Set<StoreFileReader> getStreamReaders() { List<HStore> stores = REGION.getStores(); Assert.assertEquals(1, stores.size()); HStore firstStore = stores.get(0); Assert.assertNotNull(firstStore); Collection<HStoreFile> storeFiles = firstStore.getStorefiles(); Assert.assertEquals(1, storeFiles.size()); HStoreFile firstSToreFile = storeFiles.iterator().next(); Assert.assertNotNull(firstSToreFile); return Collections.unmodifiableSet(firstSToreFile.streamReaders); }
private BlockCache setCacheProperties(HRegion region) { Iterator<HStore> strItr = region.getStores().iterator(); BlockCache cache = null; while (strItr.hasNext()) { HStore store = strItr.next(); CacheConfig cacheConf = store.getCacheConfig(); cacheConf.setCacheDataOnWrite(true); cacheConf.setEvictOnClose(true); // Use the last one cache = cacheConf.getBlockCache().get(); } return cache; }
private HStore getStoreWithName(TableName tableName) { MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads(); for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { HRegionServer hrs = rsts.get(i).getRegionServer(); for (Region region : hrs.getRegions(tableName)) { return ((HRegion) region).getStores().iterator().next(); } } return null; }
private HStore getStoreWithName(TableName tableName) { MiniHBaseCluster cluster = hbtu.getMiniHBaseCluster(); List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads(); for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { HRegionServer hrs = rsts.get(i).getRegionServer(); for (Region region : hrs.getRegions(tableName)) { return ((HRegion) region).getStores().iterator().next(); } } return null; }
private HStore getStoreWithName(TableName tableName) { MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads(); for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { HRegionServer hrs = rsts.get(i).getRegionServer(); for (HRegion region : hrs.getRegions(tableName)) { return region.getStores().iterator().next(); } } return null; }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
public static void removeCompactedFiles(Connection conn, long timeout, RegionInfo hri) throws IOException, InterruptedException { log("remove compacted files for : " + hri.getRegionNameAsString()); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(hri.getTable()); regions.stream().forEach(r -> { try { r.getStores().get(0).closeAndArchiveCompactedFiles(); } catch (IOException ioe) { LOG.error("failed in removing compacted file", ioe); } }); }
public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception { Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>(); for (HStore store : r.getStores()) { HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder(); replaceBlockCache.put(store, blockEncoder); final DataBlockEncoding inCache = DataBlockEncoding.PREFIX; final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache; ((HStore)store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk)); } majorCompaction(); // restore settings for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) { ((HStore)entry.getKey()).setDataBlockEncoderInTest(entry.getValue()); } }
@Override public void doAnAction() throws Exception { region.flush(true); // Compact regularly to avoid creating too many files and exceeding // the ulimit. region.compact(false); for (HStore store : region.getStores()) { store.closeAndArchiveCompactedFiles(); } } });
private List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
private static List<Path> findStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region: TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegions(htd.getTableName())) { for (HStore store : ((HRegion) region).getStores()) { for (HStoreFile storefile : store.getStorefiles()) { paths.add(storefile.getPath()); } } } return paths; }
@After public void teardown() throws IOException { if (region != null) { CacheConfig cacheConfig = region.getStores().get(0).getCacheConfig(); region.close(); WAL wal = region.getWAL(); if (wal != null) { wal.close(); } cacheConfig.getBlockCache().ifPresent(BlockCache::shutdown); region = null; } }
private int countDeleteMarkers(HRegion region) throws IOException { Scan s = new Scan(); s.setRaw(true); // use max versions from the store(s) s.setMaxVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions()); InternalScanner scan = region.getScanner(s); List<Cell> kvs = new ArrayList<>(); int res = 0; boolean hasMore; do { hasMore = scan.next(kvs); for (Cell kv : kvs) { if(CellUtil.isDelete(kv)) res++; } kvs.clear(); } while (hasMore); scan.close(); return res; }
@Override public boolean evaluate() throws Exception { for (HRegion region : cluster.getRegions(tn)) { for (HStore store : region.getStores()) { Collection<HStoreFile> files = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (null != files && !files.isEmpty()) { LOG.debug(region.getRegionInfo().getEncodedName() + " still has compacted files"); return false; } } } return true; } }
private static List<Path> findCompactedStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { Collection<HStoreFile> compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (compactedfiles != null) { for (HStoreFile storefile : compactedfiles) { paths.add(storefile.getPath()); } } } } return paths; }
public static void compactAndBlockUntilDone(Admin admin, HRegionServer rs, byte[] regionName) throws IOException, InterruptedException { log("Compacting region: " + Bytes.toStringBinary(regionName)); admin.majorCompactRegion(regionName); log("blocking until compaction is complete: " + Bytes.toStringBinary(regionName)); Threads.sleepWithoutInterrupt(500); outer: for (;;) { for (Store store : rs.getOnlineRegion(regionName).getStores()) { if (store.getStorefilesCount() > 1) { Threads.sleep(50); continue outer; } } break; } }