protected static Scan createBatchScan() { return new Scan().setBatch(1); }
protected static Scan createBatchSmallResultSizeScan() { return new Scan().setBatch(1).setMaxResultSize(1); }
private Scan createScan() { Scan scan = new Scan(); if (batch) { scan.setBatch(1); } if (smallResultSize) { scan.setMaxResultSize(1); } if (allowPartial) { scan.setAllowPartialResults(true); } return scan; }
protected static CINode findStartNode(Table table, byte[] startKey) throws IOException { Scan scan = new Scan(); scan.setStartRow(startKey); scan.setBatch(1); scan.addColumn(FAMILY_NAME, COLUMN_PREV); long t1 = System.currentTimeMillis(); ResultScanner scanner = table.getScanner(scan); Result result = scanner.next(); long t2 = System.currentTimeMillis(); scanner.close(); if ( result != null) { CINode node = getCINode(result, new CINode()); System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key)); return node; } System.out.println("FSR " + (t2 - t1)); return null; } protected CINode getNode(byte[] row, Table table, CINode node) throws IOException {
@Test public void testBatchDoNotAllowPartial() throws InterruptedException, ExecutionException { // we set batch to 2 and max result size to 1, then server will only returns one result per call // but we should get 2 + 1 for every row. List<Result> results = TABLE.scanAll(new Scan().setBatch(2).setMaxResultSize(1)).get(); assertEquals(2 * COUNT, results.size()); for (int i = 0; i < COUNT; i++) { Result firstTwo = results.get(2 * i); assertEquals(String.format("%02d", i), Bytes.toString(firstTwo.getRow())); assertEquals(2, firstTwo.size()); assertEquals(i, Bytes.toInt(firstTwo.getValue(FAMILY, CQS[0]))); assertEquals(2 * i, Bytes.toInt(firstTwo.getValue(FAMILY, CQS[1]))); Result secondOne = results.get(2 * i + 1); assertEquals(String.format("%02d", i), Bytes.toString(secondOne.getRow())); assertEquals(1, secondOne.size()); assertEquals(3 * i, Bytes.toInt(secondOne.getValue(FAMILY, CQS[2]))); } }
@Test public void testReversedBatchDoNotAllowPartial() throws InterruptedException, ExecutionException { // we set batch to 2 and max result size to 1, then server will only returns one result per call // but we should get 2 + 1 for every row. List<Result> results = TABLE.scanAll(new Scan().setBatch(2).setMaxResultSize(1).setReversed(true)).get(); assertEquals(2 * COUNT, results.size()); for (int i = 0; i < COUNT; i++) { int row = COUNT - i - 1; Result firstTwo = results.get(2 * i); assertEquals(String.format("%02d", row), Bytes.toString(firstTwo.getRow())); assertEquals(2, firstTwo.size()); assertEquals(row, Bytes.toInt(firstTwo.getValue(FAMILY, CQS[0]))); assertEquals(2 * row, Bytes.toInt(firstTwo.getValue(FAMILY, CQS[1]))); Result secondOne = results.get(2 * i + 1); assertEquals(String.format("%02d", row), Bytes.toString(secondOne.getRow())); assertEquals(1, secondOne.size()); assertEquals(3 * row, Bytes.toInt(secondOne.getValue(FAMILY, CQS[2]))); } } }
@Test public void testBatch() throws InterruptedException, ExecutionException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(2)).setBatch(2).setMaxResultSize(1) .setLimit(limit); List<Result> results = TABLE.scanAll(scan).get(); assertEquals(limit, results.size()); IntStream.range(0, limit).forEach(i -> { Result result = results.get(i); assertEquals(i, Bytes.toInt(result.getRow())); assertEquals(2, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); assertEquals(i, Bytes.toInt(result.getValue(FAMILY, CQS[0]))); assertEquals(2 * i, Bytes.toInt(result.getValue(FAMILY, CQS[1]))); }); }
@Test public void testBatchAllowPartial() throws InterruptedException, ExecutionException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(3)).setBatch(2).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); List<Result> results = TABLE.scanAll(scan).get(); assertEquals(3 * limit, results.size()); IntStream.range(0, 3 * limit).forEach(i -> { int key = i / 3; Result result = results.get(i); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 3; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); }); }
@Test public void testFirstKeyOnlyFilterAndBatch() throws IOException { Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); scan.setBatch(1); InternalScanner scanner = REGION.getScanner(scan); List<Cell> results = new ArrayList<>(); for (int i = 0; i < NUM_ROWS; i++) { results.clear(); scanner.next(results); assertEquals(1, results.size()); Cell cell = results.get(0); assertArrayEquals(ROWS[i], Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); } assertFalse(scanner.next(results)); scanner.close(); }
/** * Basic client side validation of HBASE-13262 */ @Test public void testScannerSeesAllRecords() throws Exception { Connection conn = TEST_UTIL.getConnection(); try (Table table = conn.getTable(TABLENAME)) { Scan s = new Scan(); s.addFamily(FAMILY); s.setMaxResultSize(-1); s.setBatch(-1); s.setCaching(500); Entry<Long,Long> entry = sumTable(table.getScanner(s)); long rowsObserved = entry.getKey(); long entriesObserved = entry.getValue(); // Verify that we see 1M rows and 9M cells assertEquals(NUM_ROWS, rowsObserved); assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); } }
@Test public void testBatchAndFilterDiffer() throws InterruptedException, ExecutionException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(3)).setBatch(2).setMaxResultSize(1) .setLimit(limit); List<Result> results = TABLE.scanAll(scan).get(); assertEquals(2 * limit, results.size()); IntStream.range(0, limit).forEach(i -> { Result result = results.get(2 * i); assertEquals(i, Bytes.toInt(result.getRow())); assertEquals(2, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); assertEquals(i, Bytes.toInt(result.getValue(FAMILY, CQS[0]))); assertEquals(2 * i, Bytes.toInt(result.getValue(FAMILY, CQS[1]))); result = results.get(2 * i + 1); assertEquals(i, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertFalse(result.mayHaveMoreCellsInRow()); assertEquals(3 * i, Bytes.toInt(result.getValue(FAMILY, CQS[2]))); }); } }
@Test public void testMayHaveMoreCellsInRowReturnsTrueAndSetBatch() throws IOException { Table table = createTestTable(TableName.valueOf(name.getMethodName()), ROWS, FAMILIES, QUALIFIERS, VALUE); Scan scan = new Scan(); scan.setBatch(1); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner scanner = table.getScanner(scan); Result result; while ((result = scanner.next()) != null) { assertTrue(result.rawCells() != null); assertEquals(1, result.rawCells().length); } }
@Test public void testRowAtomic() throws IOException, InterruptedException { byte[] row = Bytes.toBytes("row"); put(row, CQ1, Bytes.toBytes(1)); put(row, CQ2, Bytes.toBytes(2)); try (Table table = CONN.getTable(tableName); ResultScanner scanner = table.getScanner(new Scan().setBatch(1).setCaching(1))) { Result result = scanner.next(); assertEquals(1, result.rawCells().length); assertEquals(1, Bytes.toInt(result.getValue(CF, CQ1))); move(); put(row, CQ3, Bytes.toBytes(3)); result = scanner.next(); assertEquals(1, result.rawCells().length); assertEquals(2, Bytes.toInt(result.getValue(CF, CQ2))); assertNull(scanner.next()); } }
/** * Basic client side validation of HBASE-13262 */ @Test public void testSmallScannerSeesAllRecords() throws Exception { Connection conn = TEST_UTIL.getConnection(); try (Table table = conn.getTable(TABLENAME)) { Scan s = new Scan(); s.setSmall(true); s.addFamily(FAMILY); s.setMaxResultSize(-1); s.setBatch(-1); s.setCaching(500); Entry<Long,Long> entry = sumTable(table.getScanner(s)); long rowsObserved = entry.getKey(); long entriesObserved = entry.getValue(); // Verify that we see 1M rows and 9M cells assertEquals(NUM_ROWS, rowsObserved); assertEquals(NUM_ROWS * NUM_COLS, entriesObserved); } }
@Override protected boolean doAction() throws Exception { ResultScanner rs = null; try { Scan s = new Scan(); s.setBatch(2); s.addFamily(FAMILY); s.setFilter(new KeyOnlyFilter()); s.setMaxVersions(1); rs = table.getScanner(s); Result result = rs.next(); return result != null && result.size() > 0; } finally { if (rs != null) { rs.close(); } } } @Override
@Test public void testScanSize() throws Exception { doNPuts(100, true); // batch put Scan s = new Scan(); s.setBatch(1); s.setCaching(1); ResultScanner resultScanners = table.getScanner(s); for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { Result result = resultScanners.next(); assertNotNull(result); assertEquals(1, result.size()); } numScanNext += NUM_SCAN_NEXT; assertRegionMetrics("scanCount", NUM_SCAN_NEXT); if (TABLES_ON_MASTER) { assertCounter("ScanSize_num_ops", numScanNext); } }
@Test public void testScanTime() throws Exception { doNPuts(100, true); Scan s = new Scan(); s.setBatch(1); s.setCaching(1); ResultScanner resultScanners = table.getScanner(s); for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) { Result result = resultScanners.next(); assertNotNull(result); assertEquals(1, result.size()); } numScanNext += NUM_SCAN_NEXT; assertRegionMetrics("scanCount", NUM_SCAN_NEXT); if (TABLES_ON_MASTER) { assertCounter("ScanTime_num_ops", numScanNext); } }
@Test public void testFilterAllRecords() throws IOException { Scan scan = new Scan(); scan.setBatch(1); scan.setCaching(1); // Filter out any records scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0]))); try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) { try (ResultScanner s = table.getScanner(scan)) { assertNull(s.next()); } } }
@Test public void testBatch() throws IOException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(2)).setBatch(2).setMaxResultSize(1) .setLimit(limit); try (Table table = UTIL.getConnection().getTable(TABLE_NAME); ResultScanner scanner = table.getScanner(scan)) { for (int i = 0; i < limit; i++) { Result result = scanner.next(); assertEquals(i, Bytes.toInt(result.getRow())); assertEquals(2, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); assertEquals(i, Bytes.toInt(result.getValue(FAMILY, CQS[0]))); assertEquals(2 * i, Bytes.toInt(result.getValue(FAMILY, CQS[1]))); } assertNull(scanner.next()); } }
@Test public void testBatchAllowPartial() throws IOException { int limit = 5; Scan scan = new Scan().setFilter(new ColumnCountOnRowFilter(3)).setBatch(2).setMaxResultSize(1) .setAllowPartialResults(true).setLimit(limit); try (Table table = UTIL.getConnection().getTable(TABLE_NAME); ResultScanner scanner = table.getScanner(scan)) { for (int i = 0; i < 3 * limit; i++) { int key = i / 3; Result result = scanner.next(); assertEquals(key, Bytes.toInt(result.getRow())); assertEquals(1, result.size()); assertTrue(result.mayHaveMoreCellsInRow()); int cqIndex = i % 3; assertEquals(key * (cqIndex + 1), Bytes.toInt(result.getValue(FAMILY, CQS[cqIndex]))); } assertNull(scanner.next()); } }