request.withExclusiveStartKey(nextKey) .withAttributesToGet(attributesToGet) .withLimit(limit) .withTableName(tableName) .withConsistentRead(isConsistentRead);
private ScanResult scanWithRetries(Map<String, AttributeValue> lastEvaluatedScanKey) { ScanRequest scanRequest = new ScanRequest(). withTableName(this.dynamoTableName). withConsistentRead(true). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL). withLimit(100). withExclusiveStartKey(lastEvaluatedScanKey); for (int tryNumber = 1; ; tryNumber++) { try { ScanResult scanResult = dynamoDBClient.scan(scanRequest); return scanResult; } catch (ProvisionedThroughputExceededException e) { if (tryNumber == 3) { throw e; } } catch (InternalServerErrorException e) { if (tryNumber == 3) { throw e; } } try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } }
protected ScanRequest createScanRequest() { return new ScanRequest().withTableName(tableName) .withConsistentRead(forceConsistentRead) .withLimit(client.scanLimit(tableName)) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); } AbstractDynamoDbStore(final DynamoDBStoreManager manager, final String prefix, final String storeName) {
/** * Generate a list of attribute names found in the Aggregator's dynamo * table. Assumes that all Items in the Aggregator table are of the same * structure. * * @param dynamoClient Dynamo DB Client to use for connection to Dynamo DB. * @param dynamoTable The Dynamo Table for the Aggregator * @return A list of attribute names from the Dynamo table * @throws Exception */ protected List<String> getDictionaryEntry() throws Exception { // get a list of all columns in the table, with keys first List<String> columns = new ArrayList<>(); List<KeySchemaElement> keys = dynamoClient.describeTable(this.tableName).getTable().getKeySchema(); for (KeySchemaElement key : keys) { columns.add(key.getAttributeName()); } ScanResult scan = dynamoClient.scan(new ScanRequest().withTableName(this.tableName).withSelect( Select.ALL_ATTRIBUTES).withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) { for (String s : map.keySet()) { if (!columns.contains(s)) columns.add(s); } } return columns; }
.withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) {
ScanRequest scanRequest = new ScanRequest() .withTableName(tableName) .withLimit(itemLimit) .withExclusiveStartKey(exclusiveStartKey) .withTotalSegments(totalSegments)
scanRequest.withLimit(limit);
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
ScanRequest scanRequest = new ScanRequest(). withTableName(getTablNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withLimit(this.limit); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request);
public RetryResult<ScanResult> scanTable( String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) { final ScanRequest scanRequest = new ScanRequest(tableName) .withExclusiveStartKey(exclusiveStartKey) .withLimit(Ints.checkedCast(limit)) .withSegment(segment) .withTotalSegments(totalSegments) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); if (dynamoDBQueryFilter != null) { Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter(); if (!scanFilter.isEmpty()) { scanRequest.setScanFilter(scanFilter); } } RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() { @Override public ScanResult call() { log.debug("Executing DynamoDB scan: " + scanRequest); return dynamoDB.scan(scanRequest); } }, reporter, PrintCounter.DynamoDBReadThrottle); return retryResult; }
.withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withLimit(this.limit); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request);
ScanRequest scanRequest = new ScanRequest(). withTableName(getTableNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
ScanRequest scanRequest = new ScanRequest(). withTableName(getTableNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);
@Override public int count(final Credentials credentials, final String table, final Map<String, Condition> conditions) { final AmazonDynamoDB aws = credentials.aws(); try { final ScanRequest request = new ScanRequest() .withTableName(table) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withSelect(Select.COUNT) .withLimit(Integer.MAX_VALUE); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); final int count = result.getCount(); Logger.info( this, // @checkstyle LineLength (1 line) "#total(): COUNT=%d in '%s' using %s, %s, in %[ms]s", count, request.getTableName(), request.getFilterExpression(), new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return count; } finally { aws.shutdown(); } }
public ScanRequest copyScanRequest(ScanRequest request) { return new ScanRequest() .withTableName(request.getTableName()) .withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withLimit(request.getLimit()) .withConsistentRead(request.getConsistentRead()); } }
@Override public int count(final Credentials credentials, final String table, final Map<String, Condition> conditions) { final AmazonDynamoDB aws = credentials.aws(); try { final ScanRequest request = new ScanRequest() .withTableName(table) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withSelect(Select.COUNT) .withLimit(Integer.MAX_VALUE); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); final int count = result.getCount(); Logger.info( this, // @checkstyle LineLength (1 line) "#total(): COUNT=%d in '%s' using %s, %s, in %[ms]s", count, request.getTableName(), request.getFilterExpression(), new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return count; } finally { aws.shutdown(); } }
/** * Begins to pipe the log results by parallel scanning the table and the * consumer writing the results. */ public void pipe(final AbstractLogConsumer consumer) throws ExecutionException, InterruptedException { final DynamoDBTableScan scanner = new DynamoDBTableScan(rateLimit, client); final ScanRequest request = new ScanRequest().withTableName(tableName) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withLimit(BootstrapConstants.SCAN_LIMIT) .withConsistentRead(consistentScan); final ParallelScanExecutor scanService = scanner .getParallelScanCompletionService(request, numSegments, threadPool, section, totalSections); while (!scanService.finished()) { SegmentedScanResult result = scanService.grab(); consumer.writeResult(result); } shutdown(true); consumer.shutdown(true); }
public static ScanRequest copyScanRequest(final ScanRequest request) { return new ScanRequest().withAttributesToGet(request.getAttributesToGet()) .withScanFilter(request.getScanFilter()) .withConditionalOperator(request.getConditionalOperator()) .withExclusiveStartKey(request.getExclusiveStartKey()) .withExpressionAttributeNames(request.getExpressionAttributeNames()) .withExpressionAttributeValues(cloneItem(request.getExpressionAttributeValues())) .withFilterExpression(request.getFilterExpression()) .withIndexName(request.getIndexName()).withLimit(request.getLimit()) .withProjectionExpression(request.getProjectionExpression()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withScanFilter(request.getScanFilter()).withSelect(request.getSelect()) .withTableName(request.getTableName()).withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()); }