public boolean throttleCompaction(long compactionSize) { return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize); }
@Override public boolean shouldPerformMajorCompaction() throws IOException { for (HStoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile {} has null Reader", sf); return false; } } return storeEngine.getCompactionPolicy().shouldPerformMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
@Test public void testCustomParts() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); HStore mockStore = Mockito.mock(HStore.class); StoreEngine<?, ?, ?, ?> se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); Assert.assertTrue(se.getCompactor() instanceof DummyCompactor); } }
void compactEquals(List<HStoreFile> candidates, boolean forcemajor, boolean isOffPeak, long... expected) throws IOException { store.forceMajor = forcemajor; // Test Default compactions CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy()).selectCompaction( candidates, new ArrayList<>(), false, isOffPeak, forcemajor); List<HStoreFile> actual = new ArrayList<>(result.getFiles()); if (isOffPeak && !forcemajor) { Assert.assertTrue(result.isOffPeak()); } Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); store.forceMajor = false; } }
protected void compactEquals(long now, ArrayList<HStoreFile> candidates, long[] expectedFileSizes, long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException { ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(timeMachine); timeMachine.setValue(now); DateTieredCompactionRequest request; DateTieredCompactionPolicy policy = (DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy(); if (isMajor) { for (HStoreFile file : candidates) { ((MockHStoreFile) file).setIsMajor(true); } assertEquals(toCompact, policy.shouldPerformMajorCompaction(candidates)); request = (DateTieredCompactionRequest) policy.selectMajorCompaction(candidates); } else { assertEquals(toCompact, policy.needsCompaction(candidates, ImmutableList.of())); request = (DateTieredCompactionRequest) policy.selectMinorCompaction(candidates, false, false); } List<HStoreFile> actual = Lists.newArrayList(request.getFiles()); assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual))); assertEquals(Arrays.toString(expectedBoundaries), Arrays.toString(request.getBoundaries().toArray())); } }
@Test public void testOffPeakCompactionRatio() throws IOException { /* * NOTE: these tests are specific to describe the implementation of the * current compaction algorithm. Developed to ensure that refactoring * doesn't implicitly alter this. */ // set an off-peak compaction threshold this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F); store.storeEngine.getCompactionPolicy().setConf(this.conf); // Test with and without the flag. compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1); compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1); }
hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1; conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1; conf.setFloat(strPrefix + "ratio.offpeak", (float)newOffPeakCompactionRatio); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newOffPeakCompactionRatio, hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10; conf.setLong("hbase.regionserver.thread.compaction.throttle", newThrottlePoint); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newThrottlePoint, hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1; conf.setLong(strPrefix + "min", newMinFilesToCompact); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newMinFilesToCompact, hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1; conf.setLong(strPrefix + "max", newMaxFilesToCompact);
@Test public void testCompactionEmptyHFile() throws IOException { // Set TTL ScanInfo oldScanInfo = store.getScanInfo(); ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600, oldScanInfo.getKeepDeletedCells()); store.setScanInfo(newScanInfo); // Do not compact empty store file List<HStoreFile> candidates = sfCreate(0); for (HStoreFile file : candidates) { if (file instanceof MockHStoreFile) { MockHStoreFile mockFile = (MockHStoreFile) file; mockFile.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, -1, -1)); mockFile.setEntries(0); } } // Test Default compactions CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine .getCompactionPolicy()).selectCompaction(candidates, new ArrayList<>(), false, false, false); Assert.assertTrue(result.getFiles().isEmpty()); store.setScanInfo(oldScanInfo); } }
s.storeEngine.getCompactionPolicy().setConf(conf); try { createStoreFile(r); c = (RatioBasedCompactionPolicy)s.storeEngine.getCompactionPolicy(); Collection<HStoreFile> storeFiles = s.getStorefiles(); long mcTime = c.getNextMajorCompactTime(storeFiles);
store.storeEngine.getCompactionPolicy().setConf(conf); compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1); this.conf.setLong("hbase.hstore.compaction.min.size", minSize); store.storeEngine.getCompactionPolicy().setConf(conf); conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1); conf.setFloat("hbase.hregion.majorcompaction.jitter", 0); store.storeEngine.getCompactionPolicy().setConf(conf); try {
@Override public boolean throttleCompaction(long compactionSize) { return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize); }
@Test public void testCustomParts() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, DummyCompactionPolicy.class.getName()); conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY, DummyStoreFlusher.class.getName()); HStore mockStore = Mockito.mock(HStore.class); StoreEngine<?, ?, ?, ?> se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR); Assert.assertTrue(se instanceof DefaultStoreEngine); Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy); Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher); Assert.assertTrue(se.getCompactor() instanceof DummyCompactor); } }
void compactEquals(List<HStoreFile> candidates, boolean forcemajor, boolean isOffPeak, long... expected) throws IOException { store.forceMajor = forcemajor; // Test Default compactions CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine.getCompactionPolicy()).selectCompaction( candidates, new ArrayList<>(), false, isOffPeak, forcemajor); List<HStoreFile> actual = new ArrayList<>(result.getFiles()); if (isOffPeak && !forcemajor) { Assert.assertTrue(result.isOffPeak()); } Assert.assertEquals(Arrays.toString(expected), Arrays.toString(getSizes(actual))); store.forceMajor = false; } }
@Override public boolean isMajorCompaction() throws IOException { for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile " + sf + " has null Reader"); return false; } } return storeEngine.getCompactionPolicy().isMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
@Test public void testOffPeakCompactionRatio() throws IOException { /* * NOTE: these tests are specific to describe the implementation of the * current compaction algorithm. Developed to ensure that refactoring * doesn't implicitly alter this. */ // set an off-peak compaction threshold this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F); store.storeEngine.getCompactionPolicy().setConf(this.conf); // Test with and without the flag. compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1); compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1); }
protected void compactEquals(long now, ArrayList<HStoreFile> candidates, long[] expectedFileSizes, long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException { ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(timeMachine); timeMachine.setValue(now); DateTieredCompactionRequest request; DateTieredCompactionPolicy policy = (DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy(); if (isMajor) { for (HStoreFile file : candidates) { ((MockHStoreFile) file).setIsMajor(true); } assertEquals(toCompact, policy.shouldPerformMajorCompaction(candidates)); request = (DateTieredCompactionRequest) policy.selectMajorCompaction(candidates); } else { assertEquals(toCompact, policy.needsCompaction(candidates, ImmutableList.of())); request = (DateTieredCompactionRequest) policy.selectMinorCompaction(candidates, false, false); } List<HStoreFile> actual = Lists.newArrayList(request.getFiles()); assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual))); assertEquals(Arrays.toString(expectedBoundaries), Arrays.toString(request.getBoundaries().toArray())); } }
hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio() + 0.1; conf.setFloat(strPrefix + "ratio", (float)newCompactionRatio); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatio(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak() + 0.1; conf.setFloat(strPrefix + "ratio.offpeak", (float)newOffPeakCompactionRatio); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newOffPeakCompactionRatio, hstore.getStoreEngine().getCompactionPolicy().getConf().getCompactionRatioOffPeak(), 0.00001); hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint() + 10; conf.setLong("hbase.regionserver.thread.compaction.throttle", newThrottlePoint); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newThrottlePoint, hstore.getStoreEngine().getCompactionPolicy().getConf().getThrottlePoint()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact() + 1; conf.setLong(strPrefix + "min", newMinFilesToCompact); rs1.getConfigurationManager().notifyAllObservers(conf); assertEquals(newMinFilesToCompact, hstore.getStoreEngine().getCompactionPolicy().getConf().getMinFilesToCompact()); hstore.getStoreEngine().getCompactionPolicy().getConf().getMaxFilesToCompact() + 1; conf.setLong(strPrefix + "max", newMaxFilesToCompact);
@Test public void testCompactionEmptyHFile() throws IOException { // Set TTL ScanInfo oldScanInfo = store.getScanInfo(); ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600, oldScanInfo.getKeepDeletedCells()); store.setScanInfo(newScanInfo); // Do not compact empty store file List<HStoreFile> candidates = sfCreate(0); for (HStoreFile file : candidates) { if (file instanceof MockHStoreFile) { MockHStoreFile mockFile = (MockHStoreFile) file; mockFile.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, -1, -1)); mockFile.setEntries(0); } } // Test Default compactions CompactionRequestImpl result = ((RatioBasedCompactionPolicy) store.storeEngine .getCompactionPolicy()).selectCompaction(candidates, new ArrayList<>(), false, false, false); Assert.assertTrue(result.getFiles().isEmpty()); store.setScanInfo(oldScanInfo); } }
s.storeEngine.getCompactionPolicy().setConf(conf); try { createStoreFile(r); c = (RatioBasedCompactionPolicy)s.storeEngine.getCompactionPolicy(); Collection<HStoreFile> storeFiles = s.getStorefiles(); long mcTime = c.getNextMajorCompactTime(storeFiles);
store.storeEngine.getCompactionPolicy().setConf(conf); compactEquals(sfCreate(512,256,128,64,32,16,8,4,2,1), 4,2,1); this.conf.setLong("hbase.hstore.compaction.min.size", minSize); store.storeEngine.getCompactionPolicy().setConf(conf); conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1); conf.setFloat("hbase.hregion.majorcompaction.jitter", 0); store.storeEngine.getCompactionPolicy().setConf(conf); try {