/** * Restart from survivable exceptions by creating a new scanner. * * @param firstRow The first row to start at. * @throws IOException When restarting fails. */ public void restart(byte[] firstRow) throws IOException { currentScan = new Scan(scan); currentScan.withStartRow(firstRow); currentScan.setScanMetricsEnabled(true); if (this.scanner != null) { if (logScannerActivity) { LOG.info("Closing the previously opened scanner object."); } this.scanner.close(); } this.scanner = this.htable.getScanner(currentScan); if (logScannerActivity) { LOG.info("Current scan=" + currentScan.toString()); timestamp = System.currentTimeMillis(); rowcount = 0; } }
public void initialize(InputSplit split, Configuration conf) throws IOException { this.scan = TableMapReduceUtil.convertStringToScan(split.getScan()); this.split = split; TableDescriptor htd = split.htd; HRegionInfo hri = this.split.getRegionInfo(); FileSystem fs = FSUtils.getCurrentFileSystem(conf); // region is immutable, this should be fine, // otherwise we have to set the thread read point scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); // disable caching of data blocks scan.setCacheBlocks(false); scan.setScanMetricsEnabled(true); scanner = new ClientSideRegionScanner(conf, fs, new Path(split.restoreDir), htd, hri, scan, null); }
private Scan getScan() { Scan scan = new Scan(); // default scan settings scan.setCacheBlocks(false); scan.setMaxVersions(1); scan.setScanMetricsEnabled(true); if (caching != null) { scan.setCaching(Integer.parseInt(caching)); } return scan; }
@Test public void testScanMetrics() throws Exception { Pair<List<Result>, ScanMetrics> pair = method.scan(new Scan().setScanMetricsEnabled(true)); List<Result> results = pair.getFirst(); assertEquals(3, results.size()); long bytes = results.stream().flatMap(r -> Arrays.asList(r.rawCells()).stream()) .mapToLong(c -> PrivateCellUtil.estimatedSerializedSizeOf(c)).sum(); ScanMetrics scanMetrics = pair.getSecond(); assertEquals(NUM_REGIONS, scanMetrics.countOfRegions.get()); assertEquals(bytes, scanMetrics.countOfBytesInResults.get()); assertEquals(NUM_REGIONS, scanMetrics.countOfRPCcalls.get()); // also assert a server side metric to ensure that we have published them into the client side // metrics. assertEquals(3, scanMetrics.countOfRowsScanned.get()); } }
@Override boolean testRow(final int i) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); } } else { scan.addFamily(familyName); } } if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } this.testScanner = asyncTable.getScanner(scan); } Result r = testScanner.next(); updateValueSize(r); return true; } }
@Override boolean testRow(final int i) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); } } else { scan.addFamily(familyName); } } if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } this.testScanner = table.getScanner(scan); } Result r = testScanner.next(); updateValueSize(r); return true; } }
private void testRowsSeenMetric(boolean async) throws Exception { // Base scan configuration Scan baseScan; baseScan = new Scan(); baseScan.setScanMetricsEnabled(true); baseScan.setAsyncPrefetch(async); testRowsSeenMetric(baseScan); // Test case that only a single result will be returned per RPC to the serer baseScan.setCaching(1); testRowsSeenMetric(baseScan); // Test case that partial results are returned from the server. At most one cell will be // contained in each response baseScan.setMaxResultSize(1); testRowsSeenMetric(baseScan); // Test case that size limit is set such that a few cells are returned per partial result from // the server baseScan.setCaching(NUM_ROWS); baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1)); testRowsSeenMetric(baseScan); }
@Test public void testRowsFilteredMetric() throws Exception { // Base scan configuration Scan baseScan; baseScan = new Scan(); baseScan.setScanMetricsEnabled(true); // Test case where scan uses default values testRowsFilteredMetric(baseScan); // Test case where at most one Result is retrieved per RPC baseScan.setCaching(1); testRowsFilteredMetric(baseScan); // Test case where size limit is very restrictive and partial results will be returned from // server baseScan.setMaxResultSize(1); testRowsFilteredMetric(baseScan); // Test a case where max result size limits response from server to only a few cells (not all // cells from the row) baseScan.setCaching(NUM_ROWS); baseScan.setMaxResultSize(getCellHeapSize() * (NUM_COLS - 1)); testRowsSeenMetric(baseScan); }
.withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
scan2.setScanMetricsEnabled(true); scan2.setCaching(numRecords+1); scanner = ht.getScanner(scan2); scan2.setScanMetricsEnabled(true); scan2.setCaching(1); scanner = ht.getScanner(scan2); scan2.setScanMetricsEnabled(true); scan2.setCaching(1); scanner = ht.getScanner(scan2); scan2.setScanMetricsEnabled(true); scan2.setCaching(1); scan2.setSmall(true); scanWithClose.setScanMetricsEnabled(true); ResultScanner scannerWithClose = ht.getScanner(scanWithClose); for (Result result : scannerWithClose.next(numRecords + 1)) {
protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO, CompareOperator.EQUAL, new BinaryComparator(valuePrefix)); list.addFilter(filter); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(FAMILY_ZERO, qualifier); } } else { scan.addFamily(FAMILY_ZERO); } scan.setFilter(list); return scan; } }
.setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); FilterList list = new FilterList(); for (int family = 0; family < opts.families; family++) {
public static ScanMetricsHolder getInstance(ReadMetricQueue readMetrics, String tableName, Scan scan, LogLevel connectionLogLevel) { if (connectionLogLevel == LogLevel.OFF && !readMetrics.isRequestMetricsEnabled()) { return NO_OP_INSTANCE; } scan.setScanMetricsEnabled(true); return new ScanMetricsHolder(readMetrics, tableName, scan); }
private static void scan(int caching, int batch, boolean small) throws IOException { int count = 0; Scan scan = new Scan() .setCaching(caching) // co ScanCacheBatchExample-1-Set Set caching and batch parameters. .setBatch(batch) .setSmall(small) .setScanMetricsEnabled(true); ResultScanner scanner = table.getScanner(scan); for (Result result : scanner) { count++; // co ScanCacheBatchExample-2-Count Count the number of Results available. } scanner.close(); ScanMetrics metrics = scan.getScanMetrics(); System.out.println("Caching: " + caching + ", Batch: " + batch + ", Small: " + small + ", Results: " + count + ", RPCs: " + metrics.countOfRPCcalls); }
private static void scan(int num, int caching, int batch, int offset, int maxResults, int maxResultSize, boolean dump) throws IOException { int count = 0; Scan scan = new Scan() .setCaching(caching) .setBatch(batch) .setRowOffsetPerColumnFamily(offset) .setMaxResultsPerColumnFamily(maxResults) .setMaxResultSize(maxResultSize) .setScanMetricsEnabled(true); ResultScanner scanner = table.getScanner(scan); System.out.println("Scan #" + num + " running..."); for (Result result : scanner) { count++; if (dump) System.out.println("Result [" + count + "]:" + result); } scanner.close(); ScanMetrics metrics = scan.getScanMetrics(); System.out.println("Caching: " + caching + ", Batch: " + batch + ", Offset: " + offset + ", maxResults: " + maxResults + ", maxSize: " + maxResultSize + ", Results: " + count + ", RPCs: " + metrics.countOfRPCcalls); }
.setRowOffsetPerColumnFamily(5) .setRowPrefixFilter(Bytes.toBytes("row_")) .setScanMetricsEnabled(true) .setSmall(true) .setReadType(ReadType.STREAM)
public static ScanMetricsHolder getInstance(ReadMetricQueue readMetrics, String tableName, Scan scan, LogLevel connectionLogLevel) { if (connectionLogLevel == LogLevel.OFF && !readMetrics.isRequestMetricsEnabled()) { return NO_OP_INSTANCE; } scan.setScanMetricsEnabled(true); return new ScanMetricsHolder(readMetrics, tableName, scan); }
public static ScanMetricsHolder getInstance(ReadMetricQueue readMetrics, String tableName, Scan scan, LogLevel connectionLogLevel) { if (connectionLogLevel == LogLevel.OFF && !readMetrics.isRequestMetricsEnabled()) { return NO_OP_INSTANCE; } scan.setScanMetricsEnabled(true); return new ScanMetricsHolder(readMetrics, tableName, scan); }
private Scan getScan() { Scan scan = new Scan(); // default scan settings scan.setCacheBlocks(false); scan.setMaxVersions(1); scan.setScanMetricsEnabled(true); if (caching != null) { scan.setCaching(Integer.parseInt(caching)); } return scan; }
.setRowOffsetPerColumnFamily(5) .setRowPrefixFilter(Bytes.toBytes("row_")) .setScanMetricsEnabled(true) .setSmall(true) .setReadType(ReadType.STREAM)