/** * Called by compaction thread and after region is opened to compact the * HStores if necessary. * * <p>This operation could block for a long time, so don't call it from a * time-sensitive thread. * * Note that no locking is necessary at this level because compaction only * conflicts with a region split, and that cannot happen because the region * server does them sequentially and not in parallel. * * @param compaction Compaction details, obtained by requestCompaction() * @param throughputController * @return whether the compaction completed */ public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController) throws IOException { return compact(compaction, store, throughputController, null); }
/** * This is a helper function that compact all the stores synchronously. * <p> * It is used by utilities and testing */ @VisibleForTesting public void compactStores() throws IOException { for (HStore s : stores.values()) { Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { compact(compaction.get(), s, NoLimitThroughputController.INSTANCE, null); } } }
@Override public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController, User user) throws IOException { try { return super.compact(compaction, store, throughputController, user); } finally { compactCount.getAndIncrement(); } }
@Override public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController, User user) throws IOException { boolean ret = super.compact(compaction, store, throughputController, user); if (ret) compactionStateChangeLatch.countDown(); return ret; } }
@Override public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController) throws IOException { boolean ret = super.compact(compaction, store, throughputController); if (ret) compactionStateChangeLatch.countDown(); return ret; }
@Override public boolean compact(CompactionContext compaction, HStore store, ThroughputController throughputController) throws IOException { try { return super.compact(compaction, store, throughputController); } finally { compactCount.getAndIncrement(); } }
/** * This is a helper function that compact the given store. * <p> * It is used by utilities and testing */ @VisibleForTesting void compactStore(byte[] family, ThroughputController throughputController) throws IOException { HStore s = getStore(family); Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { compact(compaction.get(), s, throughputController, null); } }
controller = NoLimitThroughputController.INSTANCE; compact(compaction.get(), s, controller, null);
/** * Call flushCache on all regions on all participating regionservers. * @throws IOException */ public void compact(boolean major) throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { r.compact(major); } } }
@Override public void doAnAction() throws Exception { region.flush(true); // Compact regularly to avoid creating too many files and exceeding // the ulimit. region.compact(false); for (HStore store : region.getStores()) { store.closeAndArchiveCompactedFiles(); } } });
private void compactTable(TableName tableName) throws IOException { for(JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) { for(HRegion region : t.getRegionServer().getRegions(tableName)) { region.compact(true); } } } }
private void loadFlushAndCompact(HRegion region, byte[] family) throws IOException { // create two hfiles in the region createHFileInRegion(region, family); createHFileInRegion(region, family); HStore s = region.getStore(family); int count = s.getStorefilesCount(); assertTrue("Don't have the expected store files, wanted >= 2 store files, but was:" + count, count >= 2); // compact the two files into one file to get files in the archive LOG.debug("Compacting stores"); region.compact(true); }
@Test public void testToShowNPEOnRegionScannerReseek() throws Exception { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); Put put = new Put(Bytes.toBytes("r1")); put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); region.put(put); put = new Put(Bytes.toBytes("r2")); put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); region.put(put); region.flush(true); Scan scan = new Scan(); scan.setMaxVersions(3); // open the first scanner RegionScanner scanner1 = region.getScanner(scan); System.out.println("Smallest read point:" + region.getSmallestReadPoint()); region.compact(true); scanner1.reseek(Bytes.toBytes("r2")); List<Cell> results = new ArrayList<>(); scanner1.next(results); Cell keyValue = results.get(0); Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); scanner1.close(); }
/** * delete the row, family and cell to create the del file */ private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf) throws IOException, InterruptedException { for (byte k0 : KEYS) { byte[] k = new byte[] { k0 }; // delete a family byte[] key1 = Bytes.add(k, Bytes.toBytes(0)); Delete delete1 = new Delete(key1); delete1.addFamily(family); table.delete(delete1); // delete one row byte[] key2 = Bytes.add(k, Bytes.toBytes(2)); Delete delete2 = new Delete(key2); table.delete(delete2); // delete one cell byte[] key3 = Bytes.add(k, Bytes.toBytes(4)); Delete delete3 = new Delete(key3); delete3.addColumn(family, qf); table.delete(delete3); } admin.flush(tableName); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); for (HRegion region : regions) { region.waitForFlushesAndCompactions(); region.compact(true); } } /**
/** * Call flushCache on all regions of the specified table. * @throws IOException */ public void compact(TableName tableName, boolean major) throws IOException { for (JVMClusterUtil.RegionServerThread t: this.hbaseCluster.getRegionServers()) { for(HRegion r: t.getRegionServer().getOnlineRegionsLocalContext()) { if(r.getTableDescriptor().getTableName().equals(tableName)) { r.compact(major); } } } }
region.compact(false); doGets(region);
region.compact(true);
@Test public void testCoprocessorInterface() throws IOException { TableName tableName = TableName.valueOf(name.getMethodName()); byte [][] families = { fam1, fam2, fam3 }; Configuration hc = initConfig(); HRegion region = initHRegion(tableName, name.getMethodName(), hc, new Class<?>[]{CoprocessorImpl.class}, families); for (int i = 0; i < 3; i++) { HBaseTestCase.addContent(region, fam3); region.flush(true); } region.compact(false); // HBASE-4197 Scan s = new Scan(); RegionScanner scanner = region.getCoprocessorHost().postScannerOpen(s, region.getScanner(s)); assertTrue(scanner instanceof CustomScanner); // this would throw an exception before HBASE-4197 scanner.next(new ArrayList<>()); HBaseTestingUtility.closeRegionAndWAL(region); Coprocessor c = region.getCoprocessorHost().findCoprocessor(CoprocessorImpl.class); assertTrue("Coprocessor not started", ((CoprocessorImpl)c).wasStarted()); assertTrue("Coprocessor not stopped", ((CoprocessorImpl)c).wasStopped()); assertTrue(((CoprocessorImpl)c).wasOpened()); assertTrue(((CoprocessorImpl)c).wasClosed()); assertTrue(((CoprocessorImpl)c).wasFlushed()); assertTrue(((CoprocessorImpl)c).wasCompacted()); }
@Test public void test() throws Exception { doIncrement(0); assertSum(); // we do not hack scan operation so using scan we could get the original values added into the // table. try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertTrue(r.rawCells().length > 2); } UTIL.flush(NAME); HRegion region = UTIL.getHBaseCluster().findRegionsForTable(NAME).get(0); HStore store = region.getStore(FAMILY); for (;;) { region.compact(true); if (store.getStorefilesCount() == 1) { break; } } assertSum(); // Should only have two cells after flush and major compaction try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertEquals(2, r.rawCells().length); } } }