/** * @param config never null */ private List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException("Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } List<ScanRequest> parallelScanRequests= new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
/** * @param config never null */ List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException( "Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } final List<ScanRequest> parallelScanRequests = new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { final ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
private List<Document> getNextScanResultSet() { final List<Document> returnValue = new ArrayList<Document>(); final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withAttributesToGet(attributesToGet) .withLimit(limit)
case SCAN: { final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withTableName(tableName) .withScanFilter(this.filter.toConditions())
/** * @param config never null */ private List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException("Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } List<ScanRequest> parallelScanRequests= new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
@Override public Collection<QueuedJob> readAll() { List<QueuedJob> queuedJobs = new LinkedList<>(); Map<String, AttributeValue> lastKey = null; do { LOG.debug("Fetching a page of jobs from DynamoDB!"); ScanRequest scanRequest = new ScanRequest() .withTableName(tableName) .withExclusiveStartKey(lastKey); ScanResult result = client.scan(scanRequest); for (Map<String, AttributeValue> item : result.getItems()) { queuedJobs.add(queuedJobFromMap(item)); } lastKey = result.getLastEvaluatedKey(); } while(lastKey != null); return queuedJobs; }
@Override protected synchronized Map<String, Object> loadPropertiesFromTable(String table) { Map<String, Object> propertyMap = new HashMap<String, Object>(); Map<String, AttributeValue> lastKeysEvaluated = null; do { ScanRequest scanRequest = new ScanRequest() .withTableName(table) .withExclusiveStartKey(lastKeysEvaluated); ScanResult result = dbScanWithThroughputBackOff(scanRequest); for (Map<String, AttributeValue> item : result.getItems()) { propertyMap.put(item.get(keyAttributeName.get()).getS(), item.get(valueAttributeName.get()).getS()); } lastKeysEvaluated = result.getLastEvaluatedKey(); } while (lastKeysEvaluated != null); return propertyMap; }
private ScanResult scanWithRetries(Map<String, AttributeValue> lastEvaluatedScanKey) { ScanRequest scanRequest = new ScanRequest(). withTableName(this.dynamoTableName). withConsistentRead(true). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL). withLimit(100). withExclusiveStartKey(lastEvaluatedScanKey); for (int tryNumber = 1; ; tryNumber++) { try { ScanResult scanResult = dynamoDBClient.scan(scanRequest); return scanResult; } catch (ProvisionedThroughputExceededException e) { if (tryNumber == 3) { throw e; } } catch (InternalServerErrorException e) { if (tryNumber == 3) { throw e; } } try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } }
.withExclusiveStartKey(lastKeyEvaluated));
.withTableName(tableName) .withLimit(itemLimit) .withExclusiveStartKey(exclusiveStartKey) .withTotalSegments(totalSegments) .withSegment(segment);
.withAttributesToGet(keyColumn, dateColumn, StreamAggregator.LAST_WRITE_SEQ) .withExclusiveStartKey(lastKey);
ScanRequest scanRequest = new ScanRequest() .withTableName(table) .withExclusiveStartKey(lastKeysEvaluated); ScanResult result = dbScanWithThroughputBackOff(scanRequest); for (Map<String, AttributeValue> item : result.getItems()) {
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
final ScanRequest rqst = this.request.withExclusiveStartKey( this.result.getLastEvaluatedKey() );
scanRequest = scanRequest.withExclusiveStartKey(Collections. singletonMap(Config._KEY, new AttributeValue(pager.getLastKey())));
scanRequest = scanRequest.withExclusiveStartKey(Collections. singletonMap(Config._KEY, new AttributeValue(pager.getLastKey())));
final ScanRequest rqst = this.request.withExclusiveStartKey( this.result.getLastEvaluatedKey() );
public static ScanRequest copyScanRequest(final ScanRequest request) { return new ScanRequest().withAttributesToGet(request.getAttributesToGet()) .withScanFilter(request.getScanFilter()) .withConditionalOperator(request.getConditionalOperator()) .withExclusiveStartKey(request.getExclusiveStartKey()) .withExpressionAttributeNames(request.getExpressionAttributeNames()) .withExpressionAttributeValues(cloneItem(request.getExpressionAttributeValues())) .withFilterExpression(request.getFilterExpression()) .withIndexName(request.getIndexName()).withLimit(request.getLimit()) .withProjectionExpression(request.getProjectionExpression()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withScanFilter(request.getScanFilter()).withSelect(request.getSelect()) .withTableName(request.getTableName()).withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()); }