@Test public void testGetWithDelete() throws IOException { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf1 = Bytes.toBytes("testqualifier"); byte [] val = Bytes.toBytes("testval"); long ts1 = System.nanoTime(); KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); long ts2 = ts1 + 1; KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); long ts3 = ts2 + 1; KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); memstore.add(put1, null); memstore.add(put2, null); memstore.add(put3, null); assertEquals(3, memstore.getActive().getCellsCount()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); memstore.add(del2, null); List<Cell> expected = new ArrayList<>(); expected.add(put3); expected.add(del2); expected.add(put2); expected.add(put1); assertEquals(4, memstore.getActive().getCellsCount()); int i = 0; for(Cell cell : memstore.getActive().getCellSet()) { assertEquals(expected.get(i++), cell); } }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, hmc.getActive().getCellsCount() - cellsCount); return totalLen; }
private MutableSegment generateMutableSegment(final Configuration conf, CellComparator comparator, MemStoreLAB memStoreLAB, MemStoreSizing memstoreSizing) { // TBD use configuration to set type of segment CellSet set = new CellSet(comparator); return new MutableSegment(set, comparator, memStoreLAB, memstoreSizing); }
@VisibleForTesting protected boolean shouldFlushInMemory(MutableSegment currActive, Cell cellToAdd, MemStoreSizing memstoreSizing) { long cellSize = currActive.getCellLength(cellToAdd); long segmentDataSize = currActive.getDataSize(); while (segmentDataSize + cellSize < inmemoryFlushSize || inWalReplay) { // when replaying edits from WAL there is no need in in-memory flush regardless the size // otherwise size below flush threshold try to update atomically if (currActive.compareAndSetDataSize(segmentDataSize, segmentDataSize + cellSize)) { if (memstoreSizing != null) { memstoreSizing.incMemStoreSize(cellSize, 0, 0, 0); } // enough space for cell - no need to flush return false; } segmentDataSize = currActive.getDataSize(); } // size above flush threshold return true; }
public void upsert(Cell cell, long readpoint, MemStoreSizing memStoreSizing, boolean sizeAddedPreOperation) { internalAdd(cell, false, memStoreSizing, sizeAddedPreOperation); SortedSet<Cell> ss = this.tailSet(firstCell); Iterator<Cell> it = ss.iterator(); int cellLen = getCellLength(cur); long heapSize = heapSizeChange(cur, true); long offHeapSize = offHeapSizeChange(cur, true); incMemStoreSize(-cellLen, -heapSize, -offHeapSize, -1); if (memStoreSizing != null) { memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize, 1);
@Test public void testKeepDeleteInmemstore() { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf = Bytes.toBytes("testqualifier"); byte [] val = Bytes.toBytes("testval"); long ts = System.nanoTime(); memstore.add(new KeyValue(row, fam, qf, ts, val), null); KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); memstore.add(delete, null); assertEquals(2, memstore.getActive().getCellsCount()); assertEquals(delete, memstore.getActive().first()); }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += kv.getLength(); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0); return totalLen; }
@Test public void testMultipleVersionsSimple() throws Exception { DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); byte [] row = Bytes.toBytes("testRow"); byte [] family = Bytes.toBytes("testFamily"); byte [] qf = Bytes.toBytes("testQualifier"); long [] stamps = {1,2,3}; byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2")}; KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); m.add(key0, null); m.add(key1, null); m.add(key2, null); assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3); }
@Test public void testDelete_CheckTimestampUpdated() throws IOException { byte[] row1 = Bytes.toBytes("row1"); byte[] col1 = Bytes.toBytes("col1"); byte[] col2 = Bytes.toBytes("col2"); byte[] col3 = Bytes.toBytes("col3"); // Setting up region this.region = initHRegion(tableName, method, CONF, fam1); // Building checkerList List<Cell> kvs = new ArrayList<>(); kvs.add(new KeyValue(row1, fam1, col1, null)); kvs.add(new KeyValue(row1, fam1, col2, null)); kvs.add(new KeyValue(row1, fam1, col3, null)); NavigableMap<byte[], List<Cell>> deleteMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); deleteMap.put(fam1, kvs); region.delete(deleteMap, Durability.SYNC_WAL); // extract the key values out the memstore: // This is kinda hacky, but better than nothing... long now = System.currentTimeMillis(); AbstractMemStore memstore = (AbstractMemStore)region.getStore(fam1).memstore; Cell firstCell = memstore.getActive().first(); assertTrue(firstCell.getTimestamp() <= now); now = firstCell.getTimestamp(); for (Cell cell : memstore.getActive().getCellSet()) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp(); } }
public void debug() { String msg = "active size=" + getActive().getDataSize(); msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryCompactionInProgress is "+ (inMemoryCompactionInProgress.get() ? "true" : "false"); LOG.debug(msg); }
@Override protected long heapSize() { return getActive().getHeapSize(); }
/** * Returns the first cell in the segment * @return the first cell in the segment */ @VisibleForTesting Cell first() { return this.getCellSet().first(); }
private void internalAdd(MutableSegment currentActive, final Cell toAdd, final boolean mslabUsed, MemStoreSizing memstoreSizing) { boolean sizeAddedPreOperation = sizeAddedPreOperation(); currentActive.add(toAdd, mslabUsed, memstoreSizing, sizeAddedPreOperation); setOldestEditTimeToNow(); }
protected void dump(Logger log) { getActive().dump(log); snapshot.dump(log); }
@Test public void testPutSameKey() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); this.memstore.add(kv, null); byte[] other = Bytes.toBytes("somethingelse"); KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); this.memstore.add(samekey, null); Cell found = this.memstore.getActive().first(); assertEquals(1, this.memstore.getActive().getCellsCount()); assertTrue(Bytes.toString(found.getValueArray()), CellUtil.matchingValue(samekey, found)); }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys, byte[] val) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += kv.getLength(); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0); return totalLen; }
private void flush(int storeFilessize) throws IOException{ this.store.snapshot(); flushStore(store, id++); assertEquals(storeFilessize, this.store.getStorefiles().size()); assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); }
Cell firstCell = memstore.getActive().first(); assertTrue(firstCell.getTimestamp() <= now); now = firstCell.getTimestamp(); for (Cell cell : memstore.getActive().getCellSet()) { assertTrue(cell.getTimestamp() <= now); now = cell.getTimestamp();
@Override protected long keySize() { return getActive().getDataSize(); }
@Override protected long heapSize() { // Need to consider heapOverhead of all segments in pipeline and active long h = getActive().getHeapSize(); for (Segment segment : this.pipeline.getSegments()) { h += segment.getHeapSize(); } return h; }