private void clearCompactedfiles(List<HStoreFile> filesToRemove) throws IOException { LOG.trace("Clearing the compacted file {} from this store", filesToRemove); try { lock.writeLock().lock(); this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove); } finally { lock.writeLock().unlock(); } }
@VisibleForTesting public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing) throws IOException { // ensure other threads do not attempt to archive the same files on close() archiveLock.lock(); try { lock.readLock().lock(); Collection<HStoreFile> copyCompactedfiles = null; try { Collection<HStoreFile> compactedfiles = this.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (CollectionUtils.isNotEmpty(compactedfiles)) { // Do a copy under read lock copyCompactedfiles = new ArrayList<>(compactedfiles); } else { LOG.trace("No compacted files to archive"); } } finally { lock.readLock().unlock(); } if (CollectionUtils.isNotEmpty(copyCompactedfiles)) { removeCompactedfiles(copyCompactedfiles, storeClosing); } } finally { archiveLock.unlock(); } }
private void closeCompactedFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStoreEngine().getStoreFileManager().getCompactedfiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } sf.closeStoreFile(true); store.getStoreEngine().getStoreFileManager().removeCompactedFiles(Lists.newArrayList(sf)); }
@Override public boolean evaluate() throws Exception { for (HRegion region : cluster.getRegions(tn)) { for (HStore store : region.getStores()) { Collection<HStoreFile> files = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (null != files && !files.isEmpty()) { LOG.debug(region.getRegionInfo().getEncodedName() + " still has compacted files"); return false; } } } return true; } }
private static List<Path> findCompactedStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { Collection<HStoreFile> compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (compactedfiles != null) { for (HStoreFile storefile : compactedfiles) { paths.add(storefile.getPath()); } } } } return paths; }
StoreFileManager fileManager = store.getStoreEngine().getStoreFileManager(); Collection<HStoreFile> initialCompactedFiles = fileManager.getCompactedfiles(); assertTrue(initialCompactedFiles == null || initialCompactedFiles.isEmpty());
hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1; conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1; conf.setFloat(strPrefix + "ratio.offpeak", (float)newOffPeakCompactionRatio); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newOffPeakCompactionRatio, hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10; conf.setLong("hbase.regionserver.thread.compaction.throttle", newThrottlePoint); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newThrottlePoint, hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1; conf.setLong(strPrefix + "min", newMinFilesToCompact); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newMinFilesToCompact, hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1; conf.setLong(strPrefix + "max", newMaxFilesToCompact);
assertEquals(1, region.getStoreFileList(new byte[][]{familyNameBytes}).size()); assertEquals(refSFCount+1, ((HStore)region.getStore(familyNameBytes)).getStoreEngine().getStoreFileManager() .getCompactedfiles().size()); assertEquals(1, region.getStoreFileList(new byte[][]{familyNameBytes}).size()); assertEquals(0, ((HStore)region.getStore(familyNameBytes)).getStoreEngine().getStoreFileManager() .getCompactedfiles().size());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); for(HStoreFile file : compactedfiles) { assertEquals("Refcount should be 3", 3, ((HStoreFile) file).getRefCount()); ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles().size()); while (scanCompletedCounter.get() != 3) { Thread.sleep(100); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); for (HStoreFile file : compactedfiles) { assertEquals("Refcount should be 0", 0, file.getRefCount()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles().size()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); for(HStoreFile file : compactedfiles) { assertEquals("Refcount should be 3", 0, ((HStoreFile) file).getRefCount()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());
private void closeCompactedFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStoreEngine().getStoreFileManager().getCompactedfiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } sf.closeStoreFile(true); store.getStoreEngine().getStoreFileManager().removeCompactedFiles(Lists.newArrayList(sf)); }
@Override public boolean evaluate() throws Exception { for (HRegion region : cluster.getRegions(tn)) { for (HStore store : region.getStores()) { Collection<HStoreFile> files = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (null != files && !files.isEmpty()) { LOG.debug(region.getRegionInfo().getEncodedName() + " still has compacted files"); return false; } } } return true; } }
private static List<Path> findCompactedStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { Collection<HStoreFile> compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (compactedfiles != null) { for (HStoreFile storefile : compactedfiles) { paths.add(storefile.getPath()); } } } } return paths; }
StoreFileManager fileManager = store.getStoreEngine().getStoreFileManager(); Collection<HStoreFile> initialCompactedFiles = fileManager.getCompactedfiles(); assertTrue(initialCompactedFiles == null || initialCompactedFiles.isEmpty());
hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1; conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1; conf.setFloat(strPrefix + "ratio.offpeak", (float)newOffPeakCompactionRatio); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newOffPeakCompactionRatio, hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10; conf.setLong("hbase.regionserver.thread.compaction.throttle", newThrottlePoint); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newThrottlePoint, hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1; conf.setLong(strPrefix + "min", newMinFilesToCompact); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newMinFilesToCompact, hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1; conf.setLong(strPrefix + "max", newMaxFilesToCompact);
assertEquals(1, region.getStoreFileList(new byte[][]{familyNameBytes}).size()); assertEquals(refSFCount+1, ((HStore)region.getStore(familyNameBytes)).getStoreEngine().getStoreFileManager() .getCompactedfiles().size()); assertEquals(1, region.getStoreFileList(new byte[][]{familyNameBytes}).size()); assertEquals(0, ((HStore)region.getStore(familyNameBytes)).getStoreEngine().getStoreFileManager() .getCompactedfiles().size());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); for(HStoreFile file : compactedfiles) { assertEquals("Refcount should be 3", 3, ((HStoreFile) file).getRefCount()); ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles().size()); while (scanCompletedCounter.get() != 3) { Thread.sleep(100); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); for (HStoreFile file : compactedfiles) { assertEquals("Refcount should be 0", 0, file.getRefCount()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles().size()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());
store.getStoreEngine().getStoreFileManager().getCompactedfiles(); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); for(HStoreFile file : compactedfiles) { assertEquals("Refcount should be 3", 0, ((HStoreFile) file).getRefCount()); compactedfiles = ((HStore) store).getStoreEngine().getStoreFileManager().getCompactedfiles(); assertTrue(compactedfiles.isEmpty());