@Test public void testCountOfCellsAfterFlatteningByScan() throws IOException { String[] keys1 = { "A", "B", "C" }; // A, B, C addRowsByKeysWith50Cols(memstore, keys1); // this should only flatten as there are no duplicates ((CompactingMemStore) memstore).flushInMemory(); while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); } List<KeyValueScanner> scanners = memstore.getScanners(Long.MAX_VALUE); // seek int count = 0; for(int i = 0; i < scanners.size(); i++) { scanners.get(i).seek(KeyValue.LOWESTKEY); while (scanners.get(i).next() != null) { count++; } } assertEquals("the count should be ", 150, count); for(int i = 0; i < scanners.size(); i++) { scanners.get(i).close(); } }
@Test public void testCountOfCellsAfterFlatteningByIterator() throws IOException { String[] keys1 = { "A", "B", "C" }; // A, B, C addRowsByKeysWith50Cols(memstore, keys1); // this should only flatten as there are no duplicates ((CompactingMemStore) memstore).flushInMemory(); while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); } // Just doing the cnt operation here MemStoreSegmentsIterator itr = new MemStoreMergerSegmentsIterator( ((CompactingMemStore) memstore).getImmutableSegments().getStoreSegments(), CellComparatorImpl.COMPARATOR, 10); int cnt = 0; try { while (itr.next() != null) { cnt++; } } finally { itr.close(); } assertEquals("the count should be ", 150, cnt); }
/** * @param cell Find the row that comes after this one. If null, we return the * first. * @return Next row or null if none found. */ Cell getNextRow(final Cell cell) { Cell lowest = null; List<Segment> segments = getSegments(); for (Segment segment : segments) { if (lowest == null) { lowest = getNextRow(cell, segment.getCellSet()); } else { lowest = getLowest(lowest, getNextRow(cell, segment.getCellSet())); } } return lowest; }
@Override public List<KeyValueScanner> getScanners(long readPt) throws IOException { MutableSegment activeTmp = getActive(); List<? extends Segment> pipelineList = pipeline.getSegments(); List<? extends Segment> snapshotList = snapshot.getAllSegments(); long numberOfSegments = 1L + pipelineList.size() + snapshotList.size(); // The list of elements in pipeline + the active element + the snapshot segment List<KeyValueScanner> list = createList((int) numberOfSegments); addToScanners(activeTmp, readPt, list); addToScanners(pipelineList, readPt, list); addToScanners(snapshotList, readPt, list); return list; }
@VisibleForTesting void flushInMemory() { MutableSegment currActive = getActive(); if(currActive.setInMemoryFlushed()) { flushInMemory(currActive); } inMemoryCompaction(); }
String.valueOf(compactionType)); ((MyCompactingMemStore) memstore).initiateType(compactionType, memstore.getConfiguration()); ((CompactingMemStore) memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and flatten while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); .getMemStoreSize()); assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); + 1 * oneCellOnCSLMHeapSize + 1 * oneCellOnCCMHeapSize; assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize());
} else { LOG.debug("FLUSHING TO DISK {}, store={}", getRegionServices().getRegionInfo().getEncodedName(), getFamilyName()); stopCompaction(); pushActiveToPipeline(getActive()); snapshotId = EnvironmentEdgeManager.currentTime(); pushPipelineToSnapshot(); } else { pushTailToSnapshot();
String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); int numOfCells = 8; String[] keys1 = { "A", "A", "B", "C", "D", "D", "E", "F" }; //A1, A2, B3, C4, D5, D6, E7, F8 assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten assertEquals(0, memstore.getSnapshot().getCellsCount()); long oneCellOnCCMHeapSize = assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize());
/** * Check whether anything need to be done based on the current active set size. * The method is invoked upon every addition to the active set. * For CompactingMemStore, flush the active set to the read-only memory if it's * size is above threshold * @param currActive intended segment to update * @param cellToAdd cell to be added to the segment * @param memstoreSizing object to accumulate changed size * @return true if the cell can be added to the */ private boolean checkAndAddToActiveSize(MutableSegment currActive, Cell cellToAdd, MemStoreSizing memstoreSizing) { if (shouldFlushInMemory(currActive, cellToAdd, memstoreSizing)) { if (currActive.setInMemoryFlushed()) { flushInMemory(currActive); if (setInMemoryCompactionFlag()) { // The thread is dispatched to do in-memory compaction in the background InMemoryCompactionRunnable runnable = new InMemoryCompactionRunnable(); if (LOG.isTraceEnabled()) { LOG.trace("Dispatching the MemStore in-memory flush for store " + store .getColumnFamilyName()); } getPool().execute(runnable); } } return false; } return true; }
((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).setIndexType( CompactingMemStore.IndexType.ARRAY_MAP); ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).setIndexType( CompactingMemStore.IndexType.ARRAY_MAP); ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).flushInMemory(); ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).flushInMemory(); .isMemStoreFlushingInMemory()) { Threads.sleep(10); .isMemStoreFlushingInMemory()) { Threads.sleep(10); ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).flushInMemory(); ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).flushInMemory(); .isMemStoreFlushingInMemory()) { Threads.sleep(10); .isMemStoreFlushingInMemory()) { Threads.sleep(10); - CellArrayImmutableSegment.DEEP_OVERHEAD_CAM); assertEquals(3, // active, one in pipeline, snapshot ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).getSegments().size());
long totalHeapSize = MutableSegment.DEEP_OVERHEAD + 4 * oneCellOnCSLMHeapSize; assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact assertEquals(0, memstore.getSnapshot().getCellsCount()); + 4 * oneCellOnCAHeapSize; assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize, ((CompactingMemStore)memstore).heapSize());
@Override @Test public void testGet_memstoreAndSnapShot() throws IOException { byte[] row = Bytes.toBytes("testrow"); byte[] fam = Bytes.toBytes("testfamily"); byte[] qf1 = Bytes.toBytes("testqualifier1"); byte[] qf2 = Bytes.toBytes("testqualifier2"); byte[] qf3 = Bytes.toBytes("testqualifier3"); byte[] qf4 = Bytes.toBytes("testqualifier4"); byte[] qf5 = Bytes.toBytes("testqualifier5"); byte[] val = Bytes.toBytes("testval"); //Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); //Pushing to pipeline ((CompactingMemStore)memstore).flushInMemory(); assertEquals(0, memstore.getSnapshot().getCellsCount()); //Creating a snapshot memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); //Adding value to "new" memstore assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); }
assertEquals(totalHeapSize, memstore.heapSize()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline - flatten assertEquals(3, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals(1.0, ((CompactingMemStore) memstore).getImmutableSegments().getEstimatedUniquesFrac(), 0); assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline without compaction assertEquals(4, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals(1.0, ((CompactingMemStore) memstore).getImmutableSegments().getEstimatedUniquesFrac(), 0); assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline without compaction assertEquals(8, ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells()); assertEquals((4.0 / 8.0), ((CompactingMemStore) memstore).getImmutableSegments().getEstimatedUniquesFrac(), 0); assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact int numCells = ((CompactingMemStore) memstore).getImmutableSegments().getNumOfCells(); assertTrue(4 == numCells || 11 == numCells); assertEquals(0, memstore.getSnapshot().getCellsCount());
if (toCellChunkMap) { ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); addRowsByKeysDataSize(memstore, keys1); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline should not compact while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline without flattening assertEquals(0, memstore.getSnapshot().getCellsCount()); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { Threads.sleep(10);
/**---------------------------------------------------------------------- * The request to dispatch the compaction asynchronous task. * The method returns true if compaction was successfully dispatched, or false if there * is already an ongoing compaction or no segments to compact. */ public boolean start() throws IOException { if (!compactingMemStore.hasImmutableSegments()) { // no compaction on empty pipeline return false; } // get a snapshot of the list of the segments from the pipeline, // this local copy of the list is marked with specific version versionedList = compactingMemStore.getImmutableSegments(); LOG.trace("Speculative compaction starting on {}/{}", compactingMemStore.getStore().getHRegion().getRegionInfo().getEncodedName(), compactingMemStore.getStore().getColumnFamilyName()); HStore store = compactingMemStore.getStore(); RegionCoprocessorHost cpHost = store.getCoprocessorHost(); if (cpHost != null) { cpHost.preMemStoreCompaction(store); } try { doCompaction(); } finally { if (cpHost != null) { cpHost.postMemStoreCompaction(store); } } return true; }
((CompactingMemStore) ((HStore) region.getStore(FAMILY1)).memstore).setCompositeSnapshot(false); ((CompactingMemStore) ((HStore) region.getStore(FAMILY3)).memstore).setCompositeSnapshot(false); cms1.flushInMemory(); cms3.flushInMemory(); region.flush(false); cms1.flushInMemory(); cms3.flushInMemory(); region.flush(false);
private void flushInMemory(MutableSegment currActive) { LOG.trace("IN-MEMORY FLUSH: Pushing active segment into compaction pipeline"); pushActiveToPipeline(currActive); }
Cell closestToEmpty = ((CompactingMemStore)this.memstore).getNextRow(KeyValue.LOWESTKEY); assertTrue(CellComparator.getInstance().compareRows(closestToEmpty, new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0); for (int i = 0; i < ROW_COUNT; i++) { Cell nr = ((CompactingMemStore)this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i), System.currentTimeMillis())); if (i + 1 == ROW_COUNT) {
@Override protected boolean setInMemoryCompactionFlag() { boolean rval = super.setInMemoryCompactionFlag(); if (rval) { RUNNER_COUNT.incrementAndGet(); if (LOG.isDebugEnabled()) { LOG.debug("runner count: " + RUNNER_COUNT.get()); } } return rval; } }
@Override protected boolean shouldFlushInMemory() { boolean rval = super.shouldFlushInMemory(); if (rval) { RUNNER_COUNT.incrementAndGet(); if (LOG.isDebugEnabled()) { LOG.debug("runner count: " + RUNNER_COUNT.get()); } } return rval; } }