@Test public void testRefreshStoreFiles() throws Exception { init(name.getMethodName()); flush(1); assertEquals(1, this.store.getStorefilesCount()); addStoreFile(); addStoreFile(); addStoreFile(); addStoreFile(); assertEquals(5, this.store.getStorefilesCount()); closeCompactedFile(0); archiveStoreFile(0); assertEquals(4, this.store.getStorefilesCount()); archiveStoreFile(0); archiveStoreFile(1); archiveStoreFile(2); assertEquals(1, this.store.getStorefilesCount()); archiveStoreFile(0); store.refreshStoreFiles(); assertEquals(0, this.store.getStorefilesCount());
init(this.name.getMethodName()); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); flush(1); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); flush(2); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); flush(3); assertCheck();
private void testNumberOfMemStoreScannersAfterFlush(List<Cell> inputCellsBeforeSnapshot, List<Cell> inputCellsAfterSnapshot) throws IOException { init(this.name.getMethodName() + "-" + inputCellsBeforeSnapshot.size()); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); long seqId = Long.MIN_VALUE; try (StoreScanner s = (StoreScanner) store.getScanner(new Scan(), quals, seqId)) { assertEquals(numberOfMemScannersBeforeFlush, countMemStoreScanner(s)); storeFlushCtx.flushCache(Mockito.mock(MonitoredTask.class)); storeFlushCtx.commit(Mockito.mock(MonitoredTask.class)); more = s.next(cells); cellCount += cells.size(); assertEquals(more ? numberOfMemScannersAfterFlush : 0, countMemStoreScanner(s)); } while (more); assertEquals("The number of cells added before snapshot is " + inputCellsBeforeSnapshot.size() inputCellsBeforeSnapshot.size() + inputCellsAfterSnapshot.size(), cellCount); assertEquals(0, countMemStoreScanner(s));
@Test public void testRefreshStoreFilesNotChanged() throws IOException { init(name.getMethodName()); assertEquals(0, this.store.getStorefilesCount()); // add some data, flush this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); flush(1); // add one more file addStoreFile(); HStore spiedStore = spy(store); // call first time after files changed spiedStore.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); // call second time spiedStore.refreshStoreFiles(); //ensure that replaceStoreFiles is not called if files are not refreshed verify(spiedStore, times(0)).replaceStoreFiles(null, null); }
@Test public void testReclaimChunkWhenScaning() throws IOException { init("testReclaimChunkWhenScaning"); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100; byte[] value = Bytes.toBytes("value"); store.add(createCell(qf1, ts, seqId, value), null); store.add(createCell(qf2, ts, seqId, value), null); store.add(createCell(qf3, ts, seqId, value), null); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); quals.add(qf1); flushStore(store, id++); } catch (IOException e) { throw new RuntimeException(e); byte[] newValue = Bytes.toBytes("newValue"); store.add(createCell(qf1, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf2, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf3, ts + 1, seqId + 1, newValue), null); } catch (IOException e) { throw new RuntimeException(e);
MyStore store = initMyStore(name.getMethodName(), conf, new MyStoreHook() {}); MemStoreSizing memStoreSizing = new NonThreadSafeMemStoreSizing(); long ts = System.currentTimeMillis(); store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSizing); flushStore(store, seqID); for (int i = 11; i < 20; i++) { store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSizing); flushStore(store, seqID); for (int i = 21; i < 30; i++) { store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSizing); flushStore(store, seqID); store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSizing); flushStore(store, seqID); store.add(createCell(Bytes.toBytes("row" + i), qf1, ts, seqID++, Bytes.toBytes("")), memStoreSizing); flushStore(store, seqID); storefiles2 = store.getStorefiles();
init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family) .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build()); byte[] value = Bytes.toBytes("thisisavarylargevalue"); long seqId = 100; store.add(createCell(qf1, ts, seqId, value), memStoreSizing); store.add(createCell(qf2, ts, seqId, value), memStoreSizing); store.add(createCell(qf3, ts, seqId, value), memStoreSizing); assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
long[] timestamps2 = new long[] {30,80}; init(this.name.getMethodName()); List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family); for (Cell kv : kvList1) { this.store.add(kv, null); flushStore(store, id++); List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family); for(Cell kv : kvList2) { this.store.add(kv, null);
/** * Getting data from memstore only * @throws IOException */ @Test public void testGet_FromMemStoreOnly() throws IOException { init(this.name.getMethodName()); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); //Get result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers); //Compare assertCheck(); }
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
Configuration conf = HBaseConfiguration.create(); MyStore myStore = initMyStore(name.getMethodName(), conf, new MyStoreHook(){ @Override public void getScanners(MyStore store) throws IOException { long seqId = 100; myStore.add(createCell(qf1, ts, seqId, oldValue), memStoreSizing); myStore.add(createCell(qf2, ts, seqId, oldValue), memStoreSizing); myStore.add(createCell(qf3, ts, seqId, oldValue), memStoreSizing); long snapshotId = id++; myStore.add(createCell(qf1, ts + 1, seqId + 1, currentValue), memStoreSizing); myStore.add(createCell(qf2, ts + 1, seqId + 1, currentValue), memStoreSizing); myStore.add(createCell(qf3, ts + 1, seqId + 1, currentValue), memStoreSizing); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); quals.add(qf1);
private void flush(int storeFilessize) throws IOException{ this.store.snapshot(); flushStore(store, id++); assertEquals(storeFilessize, this.store.getStorefiles().size()); assertEquals(0, ((AbstractMemStore)this.store.memstore).getActive().getCellsCount()); }
private Cell createCell(byte[] qualifier, long ts, long sequenceId, byte[] value) throws IOException { return createCell(row, qualifier, ts, sequenceId, value); }
Configuration conf = HBaseConfiguration.create(); conf.set(HStore.MEMSTORE_CLASS_NAME, MyCompactingMemStore.class.getName()); init(name.getMethodName(), conf, ColumnFamilyDescriptorBuilder.newBuilder(family) .setInMemoryCompaction(MemoryCompactionPolicy.BASIC).build()); byte[] value = Bytes.toBytes("value"); long seqId = 100; store.add(createCell(qf1, ts, seqId, value), memStoreSizing); store.add(createCell(qf2, ts, seqId, value), memStoreSizing); store.add(createCell(qf3, ts, seqId, value), memStoreSizing); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); quals.add(qf1);
@Test public void testRefreshStoreFilesNotChanged() throws IOException { init(name.getMethodName()); assertEquals(0, this.store.getStorefilesCount()); // add some data, flush this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); flush(1); // add one more file addStoreFile(); HStore spiedStore = spy(store); // call first time after files changed spiedStore.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); // call second time spiedStore.refreshStoreFiles(); //ensure that replaceStoreFiles is not called if files are not refreshed verify(spiedStore, times(0)).replaceStoreFiles(null, null); }
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
long[] timestamps2 = new long[] {30,80}; init(this.name.getMethodName()); List<Cell> kvList1 = getKeyValueSet(timestamps1,numRows, qf1, family); for (Cell kv : kvList1) { this.store.add(kv, null); flushStore(store, id++); List<Cell> kvList2 = getKeyValueSet(timestamps2,numRows, qf1, family); for(Cell kv : kvList2) { this.store.add(kv, null);
@Test public void testReclaimChunkWhenScaning() throws IOException { init("testReclaimChunkWhenScaning"); long ts = EnvironmentEdgeManager.currentTime(); long seqId = 100; byte[] value = Bytes.toBytes("value"); store.add(createCell(qf1, ts, seqId, value), null); store.add(createCell(qf2, ts, seqId, value), null); store.add(createCell(qf3, ts, seqId, value), null); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); quals.add(qf1); flushStore(store, id++); } catch (IOException e) { throw new RuntimeException(e); byte[] newValue = Bytes.toBytes("newValue"); store.add(createCell(qf1, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf2, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf3, ts + 1, seqId + 1, newValue), null); } catch (IOException e) { throw new RuntimeException(e);