/** * Used by tests * @param s Store to add edit too. * @param cell Cell to add. */ @VisibleForTesting protected void restoreEdit(HStore s, Cell cell, MemStoreSizing memstoreAccounting) { s.add(cell, memstoreAccounting); }
/** * @see #applyToMemStore(HStore, List, boolean, MemStoreSizing) */ private void applyToMemStore(HStore store, Cell cell, MemStoreSizing memstoreAccounting) throws IOException { // Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!! if (store == null) { checkFamily(CellUtil.cloneFamily(cell)); // Unreachable because checkFamily will throw exception } store.add(cell, memstoreAccounting); }
/** * @param delta If we are doing delta changes -- e.g. increment/append -- then this flag will be * set; when set we will run operations that make sense in the increment/append scenario * but that do not make sense otherwise. * @see #applyToMemStore(HStore, Cell, MemStoreSizing) */ private void applyToMemStore(HStore store, List<Cell> cells, boolean delta, MemStoreSizing memstoreAccounting) throws IOException { // Any change in how we update Store/MemStore needs to also be done in other applyToMemStore!!!! boolean upsert = delta && store.getColumnFamilyDescriptor().getMaxVersions() == 1; if (upsert) { store.upsert(cells, getSmallestReadPoint(), memstoreAccounting); } else { store.add(cells, memstoreAccounting); } }
/** * Getting data from memstore only * @throws IOException */ @Test public void testGet_FromMemStoreOnly() throws IOException { init(this.name.getMethodName()); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); //Get result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers); //Compare assertCheck(); }
/** * Getting data from memstore and files * @throws IOException */ @Test public void testGet_FromMemStoreAndFiles() throws IOException { init(this.name.getMethodName()); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); //flush flush(1); //Add more data this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); //flush flush(2); //Add more data this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null); //Get result = HBaseTestingUtility.getFromStoreFile(store, get.getRow(), qualifiers); //Need to sort the result since multiple files Collections.sort(result, CellComparatorImpl.COMPARATOR); //Compare assertCheck(); }
long seqId = 100; store.add(createCell(qf1, ts, seqId, value), memStoreSizing); store.add(createCell(qf2, ts, seqId, value), memStoreSizing); store.add(createCell(qf3, ts, seqId, value), memStoreSizing); assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); store.add(createCell(qf1, ts + 1, seqId + 1, value), memStoreSizing); assertEquals(1, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); store.add(createCell(qf1, ts + 2, seqId + 2, value), memStoreSizing); assertEquals(2, MyCompactingMemStoreWithCustomCompactor.RUNNER_COUNT.get()); conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE,
this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null); this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null);
byte[] value = Bytes.toBytes("value"); store.add(createCell(qf1, ts, seqId, value), null); store.add(createCell(qf2, ts, seqId, value), null); store.add(createCell(qf3, ts, seqId, value), null); TreeSet<byte[]> quals = new TreeSet<>(Bytes.BYTES_COMPARATOR); quals.add(qf1); byte[] newValue = Bytes.toBytes("newValue"); store.add(createCell(qf1, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf2, ts + 1, seqId + 1, newValue), null); store.add(createCell(qf3, ts + 1, seqId + 1, newValue), null); } catch (IOException e) { throw new RuntimeException(e);
@Override public Object run() throws Exception { // Make sure it worked (above is sensitive to caching details in hadoop core) FileSystem fs = FileSystem.get(conf); assertEquals(FaultyFileSystem.class, fs.getClass()); // Initialize region init(name.getMethodName(), conf); LOG.info("Adding some data"); store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null); store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null); LOG.info("Before flush, we should have no files"); Collection<StoreFileInfo> files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); assertEquals(0, files != null ? files.size() : 0); //flush try { LOG.info("Flushing"); flush(1); fail("Didn't bubble up IOE!"); } catch (IOException ioe) { assertTrue(ioe.getMessage().contains("Fault injected")); } LOG.info("After failed flush, we should still have no files!"); files = store.getRegionFileSystem().getStoreFiles(store.getColumnFamilyName()); assertEquals(0, files != null ? files.size() : 0); store.getHRegion().getWAL().close(); return null; } });
seqId = Math.max(seqId, c.getSequenceId()); inputCellsBeforeSnapshot.forEach(c -> store.add(c, null)); StoreFlushContext storeFlushCtx = store.createFlushContext(id++, FlushLifeCycleTracker.DUMMY); storeFlushCtx.prepare(); inputCellsAfterSnapshot.forEach(c -> store.add(c, null)); int numberOfMemScannersBeforeFlush = inputCellsAfterSnapshot.isEmpty() ? 1 : 2; try (StoreScanner s = (StoreScanner) store.getScanner(new Scan(), quals, seqId)) {
@Test public void testRefreshStoreFilesNotChanged() throws IOException { init(name.getMethodName()); assertEquals(0, this.store.getStorefilesCount()); // add some data, flush this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); flush(1); // add one more file addStoreFile(); HStore spiedStore = spy(store); // call first time after files changed spiedStore.refreshStoreFiles(); assertEquals(2, this.store.getStorefilesCount()); verify(spiedStore, times(1)).replaceStoreFiles(any(), any()); // call second time spiedStore.refreshStoreFiles(); //ensure that replaceStoreFiles is not called if files are not refreshed verify(spiedStore, times(0)).replaceStoreFiles(null, null); }
store.add(createCell(r0, qf1, ts, seqId, value0), memStoreSizing); store.add(createCell(r0, qf2, ts, seqId, value0), memStoreSizing); store.add(createCell(r0, qf3, ts, seqId, value0), memStoreSizing); store.add(createCell(r1, qf1, ts + 1, seqId + 1, value1), memStoreSizing); store.add(createCell(r1, qf2, ts + 1, seqId + 1, value1), memStoreSizing); store.add(createCell(r1, qf3, ts + 1, seqId + 1, value1), memStoreSizing); store.add(createCell(r2, qf1, ts + 2, seqId + 2, value2), memStoreSizing); store.add(createCell(r2, qf2, ts + 2, seqId + 2, value2), memStoreSizing); store.add(createCell(r2, qf3, ts + 2, seqId + 2, value2), memStoreSizing); store.add(createCell(r1, qf1, ts + 3, seqId + 3, value1), memStoreSizing); store.add(createCell(r1, qf2, ts + 3, seqId + 3, value1), memStoreSizing); store.add(createCell(r1, qf3, ts + 3, seqId + 3, value1), memStoreSizing); List<Cell> myList = new MyList<>(hook); Scan scan = new Scan()
@Test public void testLowestModificationTime() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); // Initialize region init(name.getMethodName(), conf); int storeFileNum = 4; for (int i = 1; i <= storeFileNum; i++) { LOG.info("Adding some data for the store file #"+i); this.store.add(new KeyValue(row, family, qf1, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf2, i, (byte[])null), null); this.store.add(new KeyValue(row, family, qf3, i, (byte[])null), null); flush(i); } // after flush; check the lowest time stamp long lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); long lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager,lowestTimeStampFromFS); // after compact; check the lowest time stamp store.compact(store.requestCompaction().get(), NoLimitThroughputController.INSTANCE, null); lowestTimeStampFromManager = StoreUtils.getLowestTimestamp(store.getStorefiles()); lowestTimeStampFromFS = getLowestTimeStampFromFS(fs, store.getStorefiles()); assertEquals(lowestTimeStampFromManager, lowestTimeStampFromFS); }
LOG.info("Adding some data for the store file #" + i); timeStamp = EnvironmentEdgeManager.currentTime(); this.store.add(new KeyValue(row, family, qf1, timeStamp, (byte[]) null), null); this.store.add(new KeyValue(row, family, qf2, timeStamp, (byte[]) null), null); this.store.add(new KeyValue(row, family, qf3, timeStamp, (byte[]) null), null); flush(i); edge.incrementTime(sleepTime);
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
LOG.info("Adding some data"); MemStoreSizing kvSize = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize); assertEquals(kvSize.getMemStoreSize(), mss); MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing(); store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2); kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null); flush(1); assertEquals(1, this.store.getStorefilesCount());