public MemStoreSegmentsIterator(int compactionKVMax) throws IOException { this.scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build(); }
int initialBatchProgress = scannerContext.getBatchProgress(); long initialSizeProgress = scannerContext.getDataSizeProgress(); long initialHeapSizeProgress = scannerContext.getHeapSizeProgress(); if (scannerContext.getKeepProgress()) { scannerContext.setProgress(initialBatchProgress, initialSizeProgress, initialHeapSizeProgress); } else { scannerContext.clearProgress(); scannerContext.setSizeLimitScope(LimitScope.BETWEEN_ROWS); scannerContext.setTimeLimitScope(LimitScope.BETWEEN_ROWS); limitScope = LimitScope.BETWEEN_ROWS; if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) { if (hasFilterRow) { throw new IncompatibleFilterException( return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); boolean moreRows = nextRow(scannerContext, current); if (!moreRows) { return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (scannerContext.checkTimeLimit(limitScope)) { return true;
public ScannerContext build() { return new ScannerContext(keepProgress, limits, trackMetrics); } }
/** * @param checkerScope The scope that the limits are being checked from * @return true when some limit is enforceable from the checker's scope and it has been reached */ boolean checkAnyLimitReached(LimitScope checkerScope) { return checkSizeLimit(checkerScope) || checkBatchLimit(checkerScope) || checkTimeLimit(checkerScope); }
Cell nextKv; boolean moreCellsInRow = false; boolean tmpKeepProgress = scannerContext.getKeepProgress(); scannerContext.setKeepProgress(true); heap.next(results, scannerContext); scannerContext.setKeepProgress(tmpKeepProgress); if (moreCellsInRow && scannerContext.checkBatchLimit(limitScope)) { return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues(); } else if (scannerContext.checkSizeLimit(limitScope)) { ScannerContext.NextState state = moreCellsInRow ? NextState.SIZE_LIMIT_REACHED_MID_ROW : NextState.SIZE_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues(); } else if (scannerContext.checkTimeLimit(limitScope)) { ScannerContext.NextState state = moreCellsInRow ? NextState.TIME_LIMIT_REACHED_MID_ROW : NextState.TIME_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues();
return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues(); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (cell == null) { close(false);// Do all cleanup except heap.close() return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (!scannerContext.hasAnyLimit(LimitScope.BETWEEN_CELLS) || matcher.currentRow() == null) { this.countPerRow = 0; matcher.setToNewRow(cell); if (!scannerContext.getKeepProgress()) { scannerContext.clearProgress(); if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) { return scannerContext.setScannerState(NextState.TIME_LIMIT_REACHED).hasMoreValues(); scannerContext.returnImmediately(); scannerContext.setLastPeekedCell(cell); topChanged = false; ScanQueryMatcher.MatchCode qcode = matcher.match(cell); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); scannerContext.incrementSizeProgress(cellSize, cell.heapSize()); scannerContext.incrementBatchProgress(1);
int initialBatchProgress = scannerContext.getBatchProgress(); long initialSizeProgress = scannerContext.getSizeProgress(); long initialTimeProgress = scannerContext.getTimeProgress(); if (scannerContext.getKeepProgress()) { scannerContext.setProgress(initialBatchProgress, initialSizeProgress, initialTimeProgress); } else { scannerContext.clearProgress(); scannerContext.setSizeLimitScope(LimitScope.BETWEEN_ROWS); scannerContext.setTimeLimitScope(LimitScope.BETWEEN_ROWS); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); boolean moreRows = nextRow(scannerContext, currentRow, offset, length); if (!moreRows) { return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) { if (hasFilterRow) { throw new IncompatibleFilterException( long timeProgress = scannerContext.getTimeProgress(); if (scannerContext.getKeepProgress()) { scannerContext.setProgress(initialBatchProgress, initialSizeProgress, initialTimeProgress); } else {
return scannerContext.setScannerState(NextState.MORE_VALUES).hasMoreValues(); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (cell == null) { close(); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (!scannerContext.hasAnyLimit(LimitScope.BETWEEN_CELLS) || matcher.row == null) { this.countPerRow = 0; matcher.setRow(row, offset, length); if (!scannerContext.getKeepProgress()) scannerContext.clearProgress(); scannerContext.updateTimeProgress(); if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) { return scannerContext.setScannerState(NextState.TIME_LIMIT_REACHED).hasMoreValues(); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); scannerContext.incrementSizeProgress(CellUtil.estimatedHeapSizeOfWithoutTags(cell)); scannerContext.incrementBatchProgress(1); return scannerContext.setScannerState(NextState.NO_MORE_VALUES).hasMoreValues(); if (scannerContext.checkBatchLimit(LimitScope.BETWEEN_CELLS)) { break LOOP; if (scannerContext.checkSizeLimit(LimitScope.BETWEEN_CELLS)) {
ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); scannerContext.setBatchProgress(0); boolean mayHaveMoreCellsInRow = scannerContext.mayHaveMoreCellsInRow(); Result r = Result.create(values, null, stale, mayHaveMoreCellsInRow); lastBlock.setValue(addSize(context, r, lastBlock.getValue())); boolean sizeLimitReached = scannerContext.checkSizeLimit(LimitScope.BETWEEN_ROWS); boolean timeLimitReached = scannerContext.checkTimeLimit(LimitScope.BETWEEN_ROWS); boolean resultsLimitReached = numOfResults >= maxResults; limitReached = sizeLimitReached || timeLimitReached || resultsLimitReached; Cell cursorCell = scannerContext.getLastPeekedCell(); if (cursorCell != null) { builder.setCursor(ProtobufUtil.toCursor(cursorCell)); Map<String, Long> metrics = scannerContext.getMetrics().getMetricsMap(); ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder();
ScannerContext.Builder contextBuilder = ScannerContext.newBuilder(true); contextBuilder.setSizeLimit(sizeScope, maxResultSize); contextBuilder.setBatchLimit(scanner.getBatch()); scannerContext.setBatchProgress(0); final boolean partial = scannerContext.partialResultFormed(); Result r = Result.create(values, null, stale, partial); lastBlock = addSize(context, r, lastBlock); boolean sizeLimitReached = scannerContext.checkSizeLimit(LimitScope.BETWEEN_ROWS); boolean timeLimitReached = scannerContext.checkTimeLimit(LimitScope.BETWEEN_ROWS); boolean rowLimitReached = i >= rows; limitReached = sizeLimitReached || timeLimitReached || rowLimitReached; Map<String, Long> metrics = scannerContext.getMetrics().getMetricsMap(); ScanMetrics.Builder metricBuilder = ScanMetrics.newBuilder(); NameInt64Pair.Builder pairBuilder = NameInt64Pair.newBuilder();
public static void incrementSizeProgress(ScannerContext sc, List<Cell> cells) { for (Cell cell : cells) { sc.incrementSizeProgress(CellUtil.estimatedSerializedSizeOf(cell), CellUtil.estimatedHeapSizeOf(cell)); } }
public static void updateTimeProgress(ScannerContext sc) { sc.updateTimeProgress(); } }
/** * @return true if more cells exist after this batch, false if scanner is done */ private boolean populateFromJoinedHeap(List<Cell> results, ScannerContext scannerContext) throws IOException { assert joinedContinuationRow != null; boolean moreValues = populateResult(results, this.joinedHeap, scannerContext, joinedContinuationRow); if (!scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) { // We are done with this row, reset the continuation. joinedContinuationRow = null; } // As the data is obtained from two independent heaps, we need to // ensure that result list is sorted, because Result relies on that. sort(results, comparator); return moreValues; }
Cell nextKv; boolean moreCellsInRow = false; boolean tmpKeepProgress = scannerContext.getKeepProgress(); scannerContext.setKeepProgress(true); heap.next(results, scannerContext); scannerContext.setKeepProgress(tmpKeepProgress); if (!moreCellsInRow) incrementCountOfRowsScannedMetric(scannerContext); if (scannerContext.checkBatchLimit(limitScope)) { return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues(); } else if (scannerContext.checkSizeLimit(limitScope)) { ScannerContext.NextState state = moreCellsInRow? NextState.SIZE_LIMIT_REACHED_MID_ROW: NextState.SIZE_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues(); } else if (scannerContext.checkTimeLimit(limitScope)) { ScannerContext.NextState state = moreCellsInRow? NextState.TIME_LIMIT_REACHED_MID_ROW: NextState.TIME_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues();
/** * @param checkerScope The scope that the limits are being checked from * @return true when some limit is enforceable from the checker's scope and it has been reached */ boolean checkAnyLimitReached(LimitScope checkerScope) { return checkSizeLimit(checkerScope) || checkBatchLimit(checkerScope) || checkTimeLimit(checkerScope); }
public static void incrementSizeProgress(ScannerContext sc, List<Cell> cells) { for (Cell cell : cells) { sc.incrementSizeProgress(CellUtil.estimatedSerializedSizeOf(cell), CellUtil.estimatedHeapSizeOf(cell)); } }
public static void updateTimeProgress(ScannerContext sc) { sc.updateTimeProgress(); } }
/** * @return true if more cells exist after this batch, false if scanner is done */ private boolean populateFromJoinedHeap(List<Cell> results, ScannerContext scannerContext) throws IOException { assert joinedContinuationRow != null; boolean moreValues = populateResult(results, this.joinedHeap, scannerContext, joinedContinuationRow.getRowArray(), joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength()); if (!scannerContext.checkAnyLimitReached(LimitScope.BETWEEN_CELLS)) { // We are done with this row, reset the continuation. joinedContinuationRow = null; } // As the data is obtained from two independent heaps, we need to // ensure that result list is sorted, because Result relies on that. Collections.sort(results, comparator); return moreValues; }
ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
public static void incrementSizeProgress(ScannerContext sc, List<Cell> cells) { for (Cell cell : cells) { sc.incrementSizeProgress(CellUtil.estimatedSerializedSizeOf(cell), CellUtil.estimatedHeapSizeOf(cell)); } }