public Optional<CompactionContext> requestCompaction() throws IOException { return requestCompaction(NO_PRIORITY, CompactionLifeCycleTracker.DUMMY, null); }
/** * This is a helper function that compact all the stores synchronously. * <p> * It is used by utilities and testing */ @VisibleForTesting public void compactStores() throws IOException { for (HStore s : stores.values()) { Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { compact(compaction.get(), s, NoLimitThroughputController.INSTANCE, null); } } }
/** * This is a helper function that compact the given store. * <p> * It is used by utilities and testing */ @VisibleForTesting void compactStore(byte[] family, ThroughputController throughputController) throws IOException { HStore s = getStore(family); Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { compact(compaction.get(), s, throughputController, null); } }
Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { ThroughputController controller = null;
private Optional<CompactionContext> selectCompaction(HRegion region, HStore store, int priority, CompactionLifeCycleTracker tracker, CompactionCompleteTracker completeTracker, User user) throws IOException { // don't even select for compaction if disableCompactions is set to true if (!isCompactionsEnabled()) { LOG.info(String.format("User has disabled compactions")); return Optional.empty(); } Optional<CompactionContext> compaction = store.requestCompaction(priority, tracker, user); if (!compaction.isPresent() && region.getRegionInfo() != null) { String reason = "Not compacting " + region.getRegionInfo().getRegionNameAsString() + " because compaction request was cancelled"; tracker.notExecuted(store, reason); completeTracker.completed(store); LOG.debug(reason); } return compaction; }
public HStore createStoreMock(String name) throws Exception { HStore store = mock(HStore.class, name); when(store.requestCompaction(anyInt(), any(), any())).then(inv -> selectCompaction()); when(store.getCompactPriority()).then(inv -> getPriority()); doAnswer(new CancelAnswer()).when(store).cancelRequestedCompaction(any()); return store; } }
store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); if (!compaction.isPresent()) { break;
/** * Test for HBASE-5920 - Test user requested major compactions always occurring */ @Test public void testNonUserMajorCompactionRequest() throws Exception { HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction().get().getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "System-requested major compaction should not occur if there are too many store files", false, request.isMajor()); }
Optional<CompactionContext> cc = store.requestCompaction(); assertTrue(cc.isPresent());
/** * Test for HBASE-5920 */ @Test public void testUserMajorCompactionRequest() throws IOException{ HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get() .getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "User-requested major compaction should always occur, even if there are too many store files", true, request.isMajor()); }
assertFalse(this.store.requestCompaction().isPresent()); Collection<HStoreFile> sfs = this.store.getStorefiles(); assertFalse(this.store.requestCompaction().isPresent());
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
@Override public CompactionContext requestCompaction(int priority, CompactionRequest baseRequest) throws IOException { return requestCompaction(priority, baseRequest, null); } @Override
@Override public CompactionContext requestCompaction() throws IOException { return requestCompaction(Store.NO_PRIORITY, null); }
public HStore createStoreMock(String name) throws Exception { HStore store = mock(HStore.class, name); when(store.requestCompaction(anyInt(), any(), any())).then(inv -> selectCompaction()); when(store.getCompactPriority()).then(inv -> getPriority()); doAnswer(new CancelAnswer()).when(store).cancelRequestedCompaction(any()); return store; } }
store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); if (!compaction.isPresent()) { break;
/** * Execute the actual compaction job. * If the compact once flag is not specified, execute the compaction until * no more compactions are needed. Uses the Configuration settings provided. */ private void compactStoreFiles(final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri, final String familyName, final boolean compactOnce, final boolean major) throws IOException { HStore store = getStore(conf, fs, tableDir, htd, hri, familyName, tmpDir); LOG.info("Compact table=" + htd.getTableName() + " region=" + hri.getRegionNameAsString() + " family=" + familyName); if (major) { store.triggerMajorCompaction(); } do { CompactionContext compaction = store.requestCompaction(Store.PRIORITY_USER, null); if (compaction == null) break; List<StoreFile> storeFiles = store.compact(compaction, NoLimitCompactionThroughputController.INSTANCE); if (storeFiles != null && !storeFiles.isEmpty()) { if (keepCompactedFiles && deleteCompacted) { for (StoreFile storeFile: storeFiles) { fs.delete(storeFile.getPath(), false); } } } } while (store.needsCompaction() && !compactOnce); }
/** * Test for HBASE-5920 - Test user requested major compactions always occurring */ @Test public void testNonUserMajorCompactionRequest() throws Exception { HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction().get().getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "System-requested major compaction should not occur if there are too many store files", false, request.isMajor()); }
/** * Test for HBASE-5920 */ @Test public void testUserMajorCompactionRequest() throws IOException{ HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get() .getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "User-requested major compaction should always occur, even if there are too many store files", true, request.isMajor()); }
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }