@Test public void testMultipleVersionsSimple() throws Exception { DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR); byte [] row = Bytes.toBytes("testRow"); byte [] family = Bytes.toBytes("testFamily"); byte [] qf = Bytes.toBytes("testQualifier"); long [] stamps = {1,2,3}; byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"), Bytes.toBytes("value2")}; KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]); KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]); KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]); m.add(key0, null); m.add(key1, null); m.add(key2, null); assertTrue("Expected memstore to hold 3 values, actually has " + m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3); }
@Test public void testKeepDeleteInmemstore() { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf = Bytes.toBytes("testqualifier"); byte [] val = Bytes.toBytes("testval"); long ts = System.nanoTime(); memstore.add(new KeyValue(row, fam, qf, ts, val), null); KeyValue delete = new KeyValue(row, fam, qf, ts, KeyValue.Type.Delete, val); memstore.add(delete, null); assertEquals(2, memstore.getActive().getCellsCount()); assertEquals(delete, memstore.getActive().first()); }
/** * Flush the memstore * @param storeFilesSize * @throws IOException */ private void flush(int storeFilesSize) throws IOException{ this.store.snapshot(); flushStore(store, id++); Assert.assertEquals(storeFilesSize, this.store.getStorefiles().size()); Assert.assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); }
private void flush(int storeFilessize) throws IOException{ this.store.snapshot(); flushStore(store, id++); assertEquals(storeFilessize, this.store.getStorefiles().size()); assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); }
@Test public void testGetWithDelete() throws IOException { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf1 = Bytes.toBytes("testqualifier"); byte [] val = Bytes.toBytes("testval"); long ts1 = System.nanoTime(); KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); long ts2 = ts1 + 1; KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); long ts3 = ts2 + 1; KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); memstore.add(put1, null); memstore.add(put2, null); memstore.add(put3, null); assertEquals(3, memstore.getActive().getCellsCount()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.Delete, val); memstore.add(del2, null); List<Cell> expected = new ArrayList<>(); expected.add(put3); expected.add(del2); expected.add(put2); expected.add(put1); assertEquals(4, memstore.getActive().getCellsCount()); int i = 0; for(Cell cell : memstore.getActive().getCellSet()) { assertEquals(expected.get(i++), cell); } }
@Test public void testGetWithDeleteColumn() throws IOException { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf1 = Bytes.toBytes("testqualifier"); byte [] val = Bytes.toBytes("testval"); long ts1 = System.nanoTime(); KeyValue put1 = new KeyValue(row, fam, qf1, ts1, val); long ts2 = ts1 + 1; KeyValue put2 = new KeyValue(row, fam, qf1, ts2, val); long ts3 = ts2 + 1; KeyValue put3 = new KeyValue(row, fam, qf1, ts3, val); memstore.add(put1, null); memstore.add(put2, null); memstore.add(put3, null); assertEquals(3, memstore.getActive().getCellsCount()); KeyValue del2 = new KeyValue(row, fam, qf1, ts2, KeyValue.Type.DeleteColumn, val); memstore.add(del2, null); List<Cell> expected = new ArrayList<>(); expected.add(put3); expected.add(del2); expected.add(put2); expected.add(put1); assertEquals(4, memstore.getActive().getCellsCount()); int i = 0; for (Cell cell : memstore.getActive().getCellSet()) { assertEquals(expected.get(i++), cell); } }
expected.add(put3); assertEquals(5, memstore.getActive().getCellsCount()); int i = 0; for (Cell cell : memstore.getActive().getCellSet()) {
@Test public void testGet_memstoreAndSnapShot() throws IOException { byte [] row = Bytes.toBytes("testrow"); byte [] fam = Bytes.toBytes("testfamily"); byte [] qf1 = Bytes.toBytes("testqualifier1"); byte [] qf2 = Bytes.toBytes("testqualifier2"); byte [] qf3 = Bytes.toBytes("testqualifier3"); byte [] qf4 = Bytes.toBytes("testqualifier4"); byte [] qf5 = Bytes.toBytes("testqualifier5"); byte [] val = Bytes.toBytes("testval"); //Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); //Creating a snapshot memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); //Adding value to "new" memstore assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam ,qf4, val), null); memstore.add(new KeyValue(row, fam ,qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); byte[] val = Bytes.toBytes(keys[i] + i); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, hmc.getActive().getCellsCount() - cellsCount); return totalLen; }
@Test public void testPutSameKey() { byte[] bytes = Bytes.toBytes(getName()); KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes); this.memstore.add(kv, null); byte[] other = Bytes.toBytes("somethingelse"); KeyValue samekey = new KeyValue(bytes, bytes, bytes, other); this.memstore.add(samekey, null); Cell found = this.memstore.getActive().first(); assertEquals(1, this.memstore.getActive().getCellsCount()); assertTrue(Bytes.toString(found.getValueArray()), CellUtil.matchingValue(samekey, found)); }
@Override @Test public void testGet_memstoreAndSnapShot() throws IOException { byte[] row = Bytes.toBytes("testrow"); byte[] fam = Bytes.toBytes("testfamily"); byte[] qf1 = Bytes.toBytes("testqualifier1"); byte[] qf2 = Bytes.toBytes("testqualifier2"); byte[] qf3 = Bytes.toBytes("testqualifier3"); byte[] qf4 = Bytes.toBytes("testqualifier4"); byte[] qf5 = Bytes.toBytes("testqualifier5"); byte[] val = Bytes.toBytes("testval"); //Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); //Pushing to pipeline ((CompactingMemStore)memstore).flushInMemory(); assertEquals(0, memstore.getSnapshot().getCellsCount()); //Creating a snapshot memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); //Adding value to "new" memstore assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); }
protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys, byte[] val) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().getDataSize(); long heapOverhead = hmc.getActive().getHeapSize(); int cellsCount = hmc.getActive().getCellsCount(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); Threads.sleep(1); // to make sure each kv gets a different ts byte[] row = Bytes.toBytes(keys[i]); KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); totalLen += Segment.getCellLength(kv); hmc.add(kv, null); LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size, hmc.getActive().getHeapSize() - heapOverhead, 0, cellsCount); return totalLen; }
/** * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased * as older keyvalues are deleted from the memstore. * @throws Exception */ @Test public void testUpsertMemstoreSize() throws Exception { Configuration conf = HBaseConfiguration.create(); memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR); MemStoreSize oldSize = memstore.size(); List<Cell> l = new ArrayList<>(); KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1); l.add(kv1); l.add(kv2); l.add(kv3); this.memstore.upsert(l, 2, null);// readpoint is 2 MemStoreSize newSize = this.memstore.size(); assert (newSize.getDataSize() > oldSize.getDataSize()); //The kv1 should be removed. assert(memstore.getActive().getCellsCount() == 2); KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); kv4.setSequenceId(1); l.clear(); l.add(kv4); this.memstore.upsert(l, 3, null); assertEquals(newSize, this.memstore.size()); //The kv2 should be removed. assert(memstore.getActive().getCellsCount() == 2); //this.memstore = null; }
assert (newSize.getDataSize() > oldSize.getDataSize()); assert (memstore.getActive().getCellsCount() == 2); assertEquals(newSize, this.memstore.size()); assert (memstore.getActive().getCellsCount() == 2);
assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount());
@Test public void testPuttingBackChunksAfterFlushing() throws IOException { byte[] row = Bytes.toBytes("testrow"); byte[] fam = Bytes.toBytes("testfamily"); byte[] qf1 = Bytes.toBytes("testqualifier1"); byte[] qf2 = Bytes.toBytes("testqualifier2"); byte[] qf3 = Bytes.toBytes("testqualifier3"); byte[] qf4 = Bytes.toBytes("testqualifier4"); byte[] qf5 = Bytes.toBytes("testqualifier5"); byte[] val = Bytes.toBytes("testval"); // Setting up memstore memstore.add(new KeyValue(row, fam, qf1, val), null); memstore.add(new KeyValue(row, fam, qf2, val), null); memstore.add(new KeyValue(row, fam, qf3, val), null); // Creating a snapshot MemStoreSnapshot snapshot = memstore.snapshot(); assertEquals(3, memstore.getSnapshot().getCellsCount()); // Adding value to "new" memstore assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount()); // close the scanners for(KeyValueScanner scanner : snapshot.getScanners()) { scanner.close(); } memstore.clearSnapshot(snapshot.getId()); int chunkCount = chunkCreator.getPoolSize(); assertTrue(chunkCount > 0); }
assertEquals(0, memstore.getActive().getCellsCount()); memstore.add(new KeyValue(row, fam, qf4, val), null); memstore.add(new KeyValue(row, fam, qf5, val), null); assertEquals(2, memstore.getActive().getCellsCount());
@Test public void testRetainsDeleteColumn() throws IOException { // add a put to memstore memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); // now process a specific delete: KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteColumn, "dont-care"); memstore.add(delete, null); assertEquals(2, memstore.getActive().getCellsCount()); assertEquals(delete, memstore.getActive().first()); }
@Test public void testRetainsDeleteVersion() throws IOException { // add a put to memstore memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); // now process a specific delete: KeyValue delete = KeyValueTestUtil.create( "row1", "fam", "a", 100, KeyValue.Type.Delete, "dont-care"); memstore.add(delete, null); assertEquals(2, memstore.getActive().getCellsCount()); assertEquals(delete, memstore.getActive().first()); }
@Test public void testRetainsDeleteFamily() throws IOException { // add a put to memstore memstore.add(KeyValueTestUtil.create("row1", "fam", "a", 100, "dont-care"), null); // now process a specific delete: KeyValue delete = KeyValueTestUtil.create("row1", "fam", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"); memstore.add(delete, null); assertEquals(2, memstore.getActive().getCellsCount()); assertEquals(delete, memstore.getActive().first()); }