private Scan createScan() { Scan scan = new Scan(); if (batch) { scan.setBatch(1); } if (smallResultSize) { scan.setMaxResultSize(1); } if (allowPartial) { scan.setAllowPartialResults(true); } return scan; }
protected Scan createScanWithSparseFilter() { return new Scan().setMaxResultSize(Long.MAX_VALUE).setCaching(Integer.MAX_VALUE) .setNeedCursorResult(true).setAllowPartialResults(true).setFilter(new SparseFilter(false)); }
protected Scan createReversedScanWithSparseFilter() { return new Scan().setMaxResultSize(Long.MAX_VALUE).setCaching(Integer.MAX_VALUE) .setReversed(true).setNeedCursorResult(true).setAllowPartialResults(true) .setFilter(new SparseFilter(true)); }
@Test public void testAllowPartial() throws InterruptedException, ExecutionException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(2)).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); List<Result> results = TABLE.scanAll(scan).get(); assertEquals(2 * limit, results.size()); IntStream.range(0, 2 * limit).forEach(i -> { int key = i / 2; Result result = results.get(i); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 2; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); }); }
@Test public void testBatchAllowPartial() throws InterruptedException, ExecutionException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(3)).setBatch(2).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); List<Result> results = TABLE.scanAll(scan).get(); assertEquals(3 * limit, results.size()); IntStream.range(0, 3 * limit).forEach(i -> { int key = i / 3; Result result = results.get(i); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 3; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); }); }
public void testExpectedNumberOfCellsPerPartialResult(Scan baseScan, int expectedNumberOfCells) throws Exception { if (LOG.isInfoEnabled()) LOG.info("groupSize:" + expectedNumberOfCells); // Use the cellHeapSize to set maxResultSize such that we know how many cells to expect back // from the call. The returned results should NOT exceed expectedNumberOfCells but may be less // than it in cases where expectedNumberOfCells is not an exact multiple of the number of // columns in the table. Scan scan = new Scan(baseScan); scan.setAllowPartialResults(true); scan.setMaxResultSize(getResultSizeForNumberOfCells(expectedNumberOfCells)); ResultScanner scanner = TABLE.getScanner(scan); Result result = null; byte[] prevRow = null; while ((result = scanner.next()) != null) { assertTrue(result.rawCells() != null); // Cases when cell count won't equal expectedNumberOfCells: // 1. Returned result is the final result needed to form the complete result for that row // 2. It is the first result we have seen for that row and thus may have been fetched as // the last group of cells that fit inside the maxResultSize assertTrue( "Result's cell count differed from expected number. result: " + result, result.rawCells().length == expectedNumberOfCells || !result.mayHaveMoreCellsInRow() || !Bytes.equals(prevRow, result.getRow())); prevRow = result.getRow(); } scanner.close(); }
/** * @return The approximate heap size of a cell in the test table. All cells should have * approximately the same heap size, so the value is cached to avoid repeating the * calculation * @throws Exception */ private long getCellHeapSize() throws Exception { if (CELL_HEAP_SIZE == -1) { // Do a partial scan that will return a single result with a single cell Scan scan = new Scan(); scan.setMaxResultSize(2); scan.setAllowPartialResults(true); ResultScanner scanner = TABLE.getScanner(scan); Result result = scanner.next(); assertTrue(result != null); assertTrue(result.rawCells() != null); assertTrue(result.rawCells().length == 1); // Estimate the cell heap size. One difference is that on server side, the KV Heap size is // estimated differently in case the cell is backed up by MSLAB byte[] (no overhead for // backing array). Thus below calculation is a bit brittle. CELL_HEAP_SIZE = result.rawCells()[0].heapSize() - (ClassSize.ARRAY + 3); if (LOG.isInfoEnabled()) LOG.info("Cell heap size: " + CELL_HEAP_SIZE); scanner.close(); } return CELL_HEAP_SIZE; }
public void testPartialResultsReassembly(Scan scanBase) throws Exception { Scan partialScan = new Scan(scanBase); partialScan.setMaxResultSize(1); partialScan.setAllowPartialResults(true); ResultScanner partialScanner = TABLE.getScanner(partialScan); Scan oneShotScan = new Scan(scanBase); oneShotScan.setMaxResultSize(Long.MAX_VALUE); ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan); ArrayList<Result> partials = new ArrayList<>(); for (int i = 0; i < NUM_ROWS; i++) { Result partialResult = null; Result completeResult = null; Result oneShotResult = null; partials.clear(); do { partialResult = partialScanner.next(); partials.add(partialResult); } while (partialResult != null && partialResult.mayHaveMoreCellsInRow()); completeResult = Result.createCompleteResult(partials); oneShotResult = oneShotScanner.next(); compareResults(completeResult, oneShotResult, null); } assertTrue(oneShotScanner.next() == null); assertTrue(partialScanner.next() == null); oneShotScanner.close(); partialScanner.close(); }
scan.withStartRow(ROW); scan.withStopRow(ROW, true); scan.setAllowPartialResults(true); scan.setMaxResultSize(2L * 1024 * 1024); scan.readVersions(1);
/** * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is * true, the scanner should not return partial results. The scanner cannot return partial results * because the entire row needs to be read for the include/exclude decision to be made */ @Test public void testNoPartialResultsWhenRowFilterPresent() throws Exception { Scan scan = new Scan(); scan.setMaxResultSize(1); scan.setAllowPartialResults(true); // If a filter hasFilter() is true then partial results should not be returned else filter // application server side would break. scan.setFilter(new RandomRowFilter(1.0f)); ResultScanner scanner = TABLE.getScanner(scan); Result r = null; while ((r = scanner.next()) != null) { assertFalse(r.mayHaveMoreCellsInRow()); } scanner.close(); }
/** * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize * @param cachingRowLimit The row limit that will be enforced through caching * @throws Exception */ public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit) throws Exception { Scan scan = new Scan(); scan.setAllowPartialResults(true); // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to // ensure that the result size we specify is not an exact multiple of the number of cells // in a row. This ensures that partial results will be returned when the result size limit // is reached before the caching limit. int cellOffset = NUM_COLS / 3; long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset); scan.setMaxResultSize(maxResultSize); scan.setCaching(cachingRowLimit); ResultScanner scanner = TABLE.getScanner(scan); ClientScanner clientScanner = (ClientScanner) scanner; Result r = null; // Approximate the number of rows we expect will fit into the specified max rsult size. If this // approximation is less than caching, then we expect that the max result size limit will be // hit before the caching limit and thus partial results may be seen boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit; while ((r = clientScanner.next()) != null) { assertTrue(!r.mayHaveMoreCellsInRow() || expectToSeePartialResults); } scanner.close(); }
@Test public void testAllowPartial() throws IOException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(2)).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); try (Table table = UTIL.getConnection().getTable(TABLE_NAME); ResultScanner scanner = table.getScanner(scan)) { for (int i = 0; i < 2 * limit; i++) { int key = i / 2; Result result = scanner.next(); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 2; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); } assertNull(scanner.next()); } }
partialScan.setAllowPartialResults(true); ResultScanner partialScanner = TABLE.getScanner(partialScan);
/** * @return The approximate heap size of a cell in the test table. All cells should have * approximately the same heap size, so the value is cached to avoid repeating the * calculation * @throws Exception */ private long getCellHeapSize() throws Exception { if (CELL_HEAP_SIZE == -1) { // Do a partial scan that will return a single result with a single cell Scan scan = new Scan(); scan.setMaxResultSize(1); scan.setAllowPartialResults(true); ResultScanner scanner = TABLE.getScanner(scan); Result result = scanner.next(); assertTrue(result != null); assertTrue(result.rawCells() != null); assertTrue(result.rawCells().length == 1); CELL_HEAP_SIZE = result.rawCells()[0].heapSize(); scanner.close(); } return CELL_HEAP_SIZE; }
@Test public void testBatchAllowPartial() throws IOException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(3)).setBatch(2).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); try (Table table = UTIL.getConnection().getTable(TABLE_NAME); ResultScanner scanner = table.getScanner(scan)) { for (int i = 0; i < 3 * limit; i++) { int key = i / 3; Result result = scanner.next(); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 3; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); } assertNull(scanner.next()); } }
@Test public void test() throws Exception { doIncrement(0); assertSum(); // we do not hack scan operation so using scan we could get the original values added into the // table. try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertTrue(r.rawCells().length > 2); } UTIL.flush(NAME); HRegion region = UTIL.getHBaseCluster().findRegionsForTable(NAME).get(0); HStore store = region.getStore(FAMILY); for (;;) { region.compact(true); if (store.getStorefilesCount() == 1) { break; } } assertSum(); // Should only have two cells after flush and major compaction try (ResultScanner scanner = TABLE.getScanner(new Scan().withStartRow(ROW) .withStopRow(ROW, true).addFamily(FAMILY).readAllVersions().setAllowPartialResults(true))) { Result r = scanner.next(); assertEquals(2, r.rawCells().length); } } }
/** * Ensure that we only see Results marked as partial when the allowPartial flag is set * @throws Exception */ @Test public void testAllowPartialResults() throws Exception { Scan scan = new Scan(); scan.setAllowPartialResults(true); scan.setMaxResultSize(1); ResultScanner scanner = TABLE.getScanner(scan); Result result = scanner.next(); assertTrue(result != null); assertTrue(result.mayHaveMoreCellsInRow()); assertTrue(result.rawCells() != null); assertTrue(result.rawCells().length == 1); scanner.close(); scan.setAllowPartialResults(false); scanner = TABLE.getScanner(scan); result = scanner.next(); assertTrue(result != null); assertTrue(!result.mayHaveMoreCellsInRow()); assertTrue(result.rawCells() != null); assertTrue(result.rawCells().length == NUM_COLS); scanner.close(); }
@Override public Void call() throws Exception { Scan scan = new Scan(); scan.setMaxResultSize(Long.MAX_VALUE); scan.setCaching(Integer.MAX_VALUE); scan.setFilter(new SparseCellFilter()); ResultScanner scanner = TABLE.getScanner(scan); int num = 0; while (scanner.next() != null) { num++; } assertEquals(1, num); scanner.close(); scan = new Scan(); scan.setMaxResultSize(Long.MAX_VALUE); scan.setCaching(Integer.MAX_VALUE); scan.setFilter(new SparseCellFilter()); scan.setAllowPartialResults(true); scanner = TABLE.getScanner(scan); num = 0; while (scanner.next() != null) { num++; } assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, num); scanner.close(); return null; } });
@Test public void testPartialResultWhenRegionMove() throws IOException { Table table = createTestTable(TableName.valueOf(name.getMethodName()), ROWS, FAMILIES, QUALIFIERS, VALUE); moveRegion(table, 1); Scan scan = new Scan(); scan.setMaxResultSize(1); scan.setAllowPartialResults(true); ResultScanner scanner = table.getScanner(scan); for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) { scanner.next(); } Result result1 = scanner.next(); assertEquals(1, result1.rawCells().length); Cell c1 = result1.rawCells()[0]; assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]); assertFalse(result1.mayHaveMoreCellsInRow()); moveRegion(table, 2); Result result2 = scanner.next(); assertEquals(1, result2.rawCells().length); Cell c2 = result2.rawCells()[0]; assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]); assertTrue(result2.mayHaveMoreCellsInRow()); moveRegion(table, 3); Result result3 = scanner.next(); assertEquals(1, result3.rawCells().length); Cell c3 = result3.rawCells()[0]; assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]); assertTrue(result3.mayHaveMoreCellsInRow()); }
scan.setAllowPartialResults(true); scan.setReversed(true); ResultScanner scanner = table.getScanner(scan);