/** * @param config never null */ private List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException("Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } List<ScanRequest> parallelScanRequests= new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
/** * @param config never null */ List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException( "Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } final List<ScanRequest> parallelScanRequests = new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { final ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
ParallelScanner getParallelScanCompletionService(final ScanRequest initialRequest) throws BackendException { final int segments = Math.max(1, clientThreadPool.getMaximumPoolSize() / maxConcurrentUsers); final ParallelScanner completion = new ParallelScanner(clientThreadPool, segments, this); for (int segment = 0; segment < segments; segment++) { // dont need to set user agent here because ExponentialBackoff.Scan // calls DynamoDbDelegate.scan which sets it final ScanRequest scanSegment = copyScanRequest(initialRequest).withTotalSegments(segments).withSegment(segment); completion.addWorker(new ScanSegmentWorker(this, scanSegment), segment); } return completion; }
/** * @param config never null */ private List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException("Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } List<ScanRequest> parallelScanRequests= new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
/** * This function copies a scan request for the number of segments and then * adds those workers to the executor service to begin scanning. * * @param totalSections * @param section * * @return <ParallelScanExecutor> the parallel scan executor to grab results * when a segment is finished. */ public ParallelScanExecutor getParallelScanCompletionService( ScanRequest initialRequest, int numSegments, Executor executor, int section, int totalSections) { final int segments = Math.max(1, numSegments); final ParallelScanExecutor completion = new ParallelScanExecutor( executor, segments); int sectionSize = segments / totalSections; int start = sectionSize * section; int end = start + sectionSize; if (section + 1 == totalSections) { end = segments; } for (int segment = start; segment < end; segment++) { ScanRequest scanSegment = copyScanRequest(initialRequest) .withTotalSegments(segments).withSegment(segment); completion.addWorker(new ScanSegmentWorker(this.client, this.rateLimiter, scanSegment), segment); } return completion; }
.withTableName(this.tableName) .withAttributesToGet(this.hashKey) .withSegment(this.workerInstance) .withTotalSegments(threads); Map<String, Set<String>> deduplicated = new HashMap<>();
.withExclusiveStartKey(exclusiveStartKey) .withTotalSegments(totalSegments) .withSegment(segment);
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
public ScanRequest copyScanRequest(ScanRequest request) { return new ScanRequest() .withTableName(request.getTableName()) .withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withLimit(request.getLimit()) .withConsistentRead(request.getConsistentRead()); } }
public static ScanRequest copyScanRequest(final ScanRequest request) { return new ScanRequest().withAttributesToGet(request.getAttributesToGet()) .withScanFilter(request.getScanFilter()) .withConditionalOperator(request.getConditionalOperator()) .withExclusiveStartKey(request.getExclusiveStartKey()) .withExpressionAttributeNames(request.getExpressionAttributeNames()) .withExpressionAttributeValues(cloneItem(request.getExpressionAttributeValues())) .withFilterExpression(request.getFilterExpression()) .withIndexName(request.getIndexName()).withLimit(request.getLimit()) .withProjectionExpression(request.getProjectionExpression()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withScanFilter(request.getScanFilter()).withSelect(request.getSelect()) .withTableName(request.getTableName()).withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()); }