/** * Atomically apply the given map of family->edits to the memstore. * This handles the consistency control on its own, but the caller * should already have locked updatesLock.readLock(). This also does * <b>not</b> check the families for validity. * * @param familyMap Map of Cells by family */ protected void applyFamilyMapToMemStore(Map<byte[], List<Cell>> familyMap, MemStoreSizing memstoreAccounting) throws IOException { for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) { byte[] family = e.getKey(); List<Cell> cells = e.getValue(); assert cells instanceof RandomAccess; region.applyToMemStore(region.getStore(family), cells, false, memstoreAccounting); } } }
/** * This is a helper function that compact the given store. * <p> * It is used by utilities and testing */ @VisibleForTesting void compactStore(byte[] family, ThroughputController throughputController) throws IOException { HStore s = getStore(family); Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { compact(compaction.get(), s, throughputController, null); } }
/** * If all stores ended up dropping their snapshots, we can safely drop the prepareFlushResult */ private void dropPrepareFlushIfPossible() { if (writestate.flushing) { boolean canDrop = true; if (prepareFlushResult.storeFlushCtxs != null) { for (Entry<byte[], StoreFlushContext> entry : prepareFlushResult.storeFlushCtxs .entrySet()) { HStore store = getStore(entry.getKey()); if (store == null) { continue; } if (store.getSnapshotSize().getDataSize() > 0) { canDrop = false; break; } } } // this means that all the stores in the region has finished flushing, but the WAL marker // may not have been written or we did not receive it yet. if (canDrop) { writestate.flushing = false; this.prepareFlushResult = null; } } }
private int countStoreFiles() throws IOException { HStore store = region.getStore(COLUMN_FAMILY); return store.getStorefilesCount(); }
public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName, byte[] storeName) throws IOException { byte[] table = Bytes.toBytes(tableName); // get the RS and region serving our table List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table); HRegion region = servingRegions.get(0); // check that we actually have some store files that were archived Store store = region.getStore(storeName); return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store); } }
private void loadFlushAndCompact(HRegion region, byte[] family) throws IOException { // create two hfiles in the region createHFileInRegion(region, family); createHFileInRegion(region, family); HStore s = region.getStore(family); int count = s.getStorefilesCount(); assertTrue("Don't have the expected store files, wanted >= 2 store files, but was:" + count, count >= 2); // compact the two files into one file to get files in the archive LOG.debug("Compacting stores"); region.compact(true); }
/** * Create a custom compaction request and be sure that we can track it through the queue, knowing * when the compaction is completed. */ @Test public void testTrackingCompactionRequest() throws Exception { // setup a compact/split thread on a mock server HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } CountDownLatch latch = new CountDownLatch(1); Tracker tracker = new Tracker(latch); thread.requestCompaction(r, store, "test custom comapction", PRIORITY_USER, tracker, null); // wait for the latch to complete. latch.await(); thread.interruptIfNecessary(); }
@Test public void testFlushedFileWithNoTags() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor(fam1)); HRegionInfo info = new HRegionInfo(tableName, null, null, false); Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); region = HBaseTestingUtility.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), htd); Put put = new Put(Bytes.toBytes("a-b-0-0")); put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); region.put(put); region.flush(true); HStore store = region.getStore(fam1); Collection<HStoreFile> storefiles = store.getStorefiles(); for (HStoreFile sf : storefiles) { assertFalse("Tags should not be present " ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
@Test public void testDelete_CheckTimestampUpdated() throws IOException { byte[] row1 = Bytes.toBytes("row1"); byte[] col1 = Bytes.toBytes("col1"); byte[] col2 = Bytes.toBytes("col2"); byte[] col3 = Bytes.toBytes("col3"); // Setting up region this.region = initHRegion(tableName, method, CONF, fam1); // Building checkerList List<Cell> kvs = new ArrayList<>(); kvs.add(new KeyValue(row1, fam1, col1, null)); kvs.add(new KeyValue(row1, fam1, col2, null)); kvs.add(new KeyValue(row1, fam1, col3, null)); NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); deleteMap.put(fam1, kvs); region.delete(deleteMap, Durability.SYNC_WAL); // extract the key values out the memstore: // This is kinda hacky, but better than nothing... long now = System.currentTimeMillis(); AbstractMemStore memstore = (AbstractMemStore)region.getStore(fam1).memstore; Cell firstCell = memstore.getActive().first(); assertTrue(firstCell.getTimestamp() <= now); now = firstCell.getTimestamp(); for (Cell cell : memstore.getActive().getCellSet()) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp(); } }
@Test public void test() throws Exception { // sleep every 10 loops to give memstore compaction enough time to finish before reaching the // flush size. doIncrement(10); assertSum(); HStore store = UTIL.getHBaseCluster().findRegionsForTable(NAME).get(0).getStore(FAMILY); // should have no store files created as we have done aggregating all in memory assertEquals(0, store.getStorefilesCount()); } }
private int count() throws IOException { int count = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
/** * Test no new Compaction requests are generated after calling stop compactions */ @Test public void testStopStartCompaction() throws IOException { // setup a compact/split thread on a mock server HRegionServer mockServer = Mockito.mock(HRegionServer.class); Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf()); CompactSplit thread = new CompactSplit(mockServer); Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread); // setup a region/store with some files HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) { createStoreFile(r); } thread.switchCompaction(false); thread .requestCompaction(r, store, "test", Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); assertEquals(false, thread.isCompactionsEnabled()); assertEquals(0, thread.getLongCompactions().getActiveCount() + thread.getShortCompactions() .getActiveCount()); thread.switchCompaction(true); assertEquals(true, thread.isCompactionsEnabled()); thread .requestCompaction(r, store, "test", Store.PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null); assertEquals(1, thread.getLongCompactions().getActiveCount() + thread.getShortCompactions() .getActiveCount()); }
HStore store = region.getStore(Bytes.toBytes("f")); long expectedStoreFilesSize = store.getStorefilesSize(); Assert.assertNotNull(store);
/** * Test for HBASE-5920 - Test user requested major compactions always occurring */ @Test public void testNonUserMajorCompactionRequest() throws Exception { HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction().get().getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "System-requested major compaction should not occur if there are too many store files", false, request.isMajor()); }
/** * Test for HBASE-5920 */ @Test public void testUserMajorCompactionRequest() throws IOException{ HStore store = r.getStore(COLUMN_FAMILY); createStoreFile(r); for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) { createStoreFile(r); } store.triggerMajorCompaction(); CompactionRequestImpl request = store.requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get() .getRequest(); assertNotNull("Expected to receive a compaction request", request); assertEquals( "User-requested major compaction should always occur, even if there are too many store files", true, request.isMajor()); }
/** * Create the mob store file */ private Path createMobStoreFile(HColumnDescriptor hcd) throws IOException { // Setting up a Store TableName tn = TableName.valueOf(TABLE); HTableDescriptor htd = new HTableDescriptor(tn); htd.addFamily(hcd); HMobStore mobStore = (HMobStore) region.getStore(hcd.getName()); KeyValue key1 = new KeyValue(ROW, hcd.getName(), QF1, 1, VALUE); KeyValue key2 = new KeyValue(ROW, hcd.getName(), QF2, 1, VALUE); KeyValue key3 = new KeyValue(ROW2, hcd.getName(), QF3, 1, VALUE2); KeyValue[] keys = new KeyValue[] { key1, key2, key3 }; int maxKeyCount = keys.length; HRegionInfo regionInfo = new HRegionInfo(tn); StoreFileWriter mobWriter = mobStore.createWriterInTmp(currentDate, maxKeyCount, hcd.getCompactionCompression(), regionInfo.getStartKey(), false); Path mobFilePath = mobWriter.getPath(); String fileName = mobFilePath.getName(); mobWriter.append(key1); mobWriter.append(key2); mobWriter.append(key3); mobWriter.close(); String targetPathName = MobUtils.formatDate(currentDate); Path targetPath = new Path(mobStore.getPath(), targetPathName); mobStore.commitFile(mobFilePath, targetPath); return new Path(targetPath, fileName); }
@Test public void testFlushedFileWithVisibilityTags() throws Exception { final byte[] qual2 = Bytes.toBytes("qual2"); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor col = new HColumnDescriptor(fam); desc.addFamily(col); TEST_UTIL.getAdmin().createTable(desc); try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { Put p1 = new Put(row1); p1.addColumn(fam, qual, value); p1.setCellVisibility(new CellVisibility(CONFIDENTIAL)); Put p2 = new Put(row1); p2.addColumn(fam, qual2, value); p2.setCellVisibility(new CellVisibility(SECRET)); RowMutations rm = new RowMutations(row1); rm.add(p1); rm.add(p2); table.mutateRow(rm); } TEST_UTIL.getAdmin().flush(tableName); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); HStore store = regions.get(0).getStore(fam); Collection<HStoreFile> storefiles = store.getStorefiles(); assertTrue(storefiles.size() > 0); for (HStoreFile storeFile : storefiles) { assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
@Test public void test() throws IOException, KeeperException, InterruptedException { long now = System.currentTimeMillis(); put(0, 100, now - 10000); assertValueEquals(0, 100); setExpireBefore(now - 5000); Thread.sleep(5000); UTIL.getAdmin().flush(NAME); assertNotExists(0, 100); put(0, 50, now - 1000); UTIL.getAdmin().flush(NAME); put(50, 100, now - 100); UTIL.getAdmin().flush(NAME); assertValueEquals(0, 100); setExpireBefore(now - 500); Thread.sleep(5000); UTIL.getAdmin().majorCompact(NAME); UTIL.waitFor(30000, () -> UTIL.getHBaseCluster().getRegions(NAME).iterator().next() .getStore(FAMILY).getStorefilesCount() == 1); assertNotExists(0, 50); assertValueEquals(50, 100); } }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = CellUtil.cloneRow(scanner.getCell()); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }
@Test public void test() throws Exception { doIncrement(0); assertSum(); // we do not hack scan operation so using scan we could get the original values added into the // table. try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertTrue(r.rawCells().length > 2); } UTIL.flush(NAME); HRegion region = UTIL.getHBaseCluster().findRegionsForTable(NAME).get(0); HStore store = region.getStore(FAMILY); for (;;) { region.compact(true); if (store.getStorefilesCount() == 1) { break; } } assertSum(); // Should only have two cells after flush and major compaction try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertEquals(2, r.rawCells().length); } } }