@Override public ScanResult scan(String tableName, java.util.List<String> attributesToGet, java.util.Map<String, Condition> scanFilter) { return scan(new ScanRequest().withTableName(tableName).withAttributesToGet(attributesToGet).withScanFilter(scanFilter)); }
/** * @param config never null */ private List<ScanRequest> createParallelScanRequestsFromExpression(Class<?> clazz, DynamoDBScanExpression scanExpression, int totalSegments, DynamoDBMapperConfig config) { if (totalSegments < 1) { throw new IllegalArgumentException("Parallel scan should have at least one scan segment."); } if (scanExpression.getExclusiveStartKey() != null) { log.info("The ExclusiveStartKey parameter specified in the DynamoDBScanExpression is ignored," + " since the individual parallel scan request on each segment is applied on a separate key scope."); } if (scanExpression.getSegment() != null || scanExpression.getTotalSegments() != null) { log.info("The Segment and TotalSegments parameters specified in the DynamoDBScanExpression are ignored."); } List<ScanRequest> parallelScanRequests= new LinkedList<ScanRequest>(); for (int segment = 0; segment < totalSegments; segment++) { ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); parallelScanRequests.add(scanRequest .withSegment(segment).withTotalSegments(totalSegments) .withExclusiveStartKey(null)); } return parallelScanRequests; }
gres = dynamoDB.getItem(greq); } catch (AmazonServiceException ex) { LOGGER.error(ex); ScanRequest req = new ScanRequest(table); req.setAttributesToGet(fields); while (count < recordcount) { req.setExclusiveStartKey(startKey); req.setLimit(recordcount - count); ScanResult res; try { res = dynamoDB.scan(req); } catch (AmazonServiceException ex) { LOGGER.error(ex); count += res.getCount(); for (Map<String, AttributeValue> items : res.getItems()) { result.add(extractResult(items)); startKey = res.getLastEvaluatedKey();
/** * Simplified method form for invoking the Scan operation. * * @see #scanAsync(ScanRequest) */ @Override public java.util.concurrent.Future<ScanResult> scanAsync(String tableName, java.util.List<String> attributesToGet) { return scanAsync(new ScanRequest().withTableName(tableName).withAttributesToGet(attributesToGet)); }
/** * Simplified method form for invoking the Scan operation. * * @see #scanAsync(ScanRequest) */ @Override public java.util.concurrent.Future<ScanResult> scanAsync(String tableName, java.util.Map<String, Condition> scanFilter) { return scanAsync(new ScanRequest().withTableName(tableName).withScanFilter(scanFilter)); }
public static ScanRequest copyScanRequest(final ScanRequest request) { return new ScanRequest().withAttributesToGet(request.getAttributesToGet()) .withScanFilter(request.getScanFilter()) .withConditionalOperator(request.getConditionalOperator()) .withExclusiveStartKey(request.getExclusiveStartKey()) .withExpressionAttributeNames(request.getExpressionAttributeNames()) .withExpressionAttributeValues(cloneItem(request.getExpressionAttributeValues())) .withFilterExpression(request.getFilterExpression()) .withIndexName(request.getIndexName()).withLimit(request.getLimit()) .withProjectionExpression(request.getProjectionExpression()) .withReturnConsumedCapacity(request.getReturnConsumedCapacity()) .withScanFilter(request.getScanFilter()).withSelect(request.getSelect()) .withTableName(request.getTableName()).withTotalSegments(request.getTotalSegments()) .withSegment(request.getSegment()); }
); attrs.addAll(keys); final ScanRequest request = new ScanRequest() .withTableName(table) .withAttributesToGet(attrs) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withLimit(this.limit); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); Logger.info( this, result.getCount(), table, result.getLastEvaluatedKey(), conditions, new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); } finally { aws.shutdown();
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getTableName() == null) ? 0 : getTableName().hashCode()); hashCode = prime * hashCode + ((getIndexName() == null) ? 0 : getIndexName().hashCode()); hashCode = prime * hashCode + ((getAttributesToGet() == null) ? 0 : getAttributesToGet().hashCode()); hashCode = prime * hashCode + ((getLimit() == null) ? 0 : getLimit().hashCode()); hashCode = prime * hashCode + ((getSelect() == null) ? 0 : getSelect().hashCode()); hashCode = prime * hashCode + ((getScanFilter() == null) ? 0 : getScanFilter().hashCode()); hashCode = prime * hashCode + ((getConditionalOperator() == null) ? 0 : getConditionalOperator().hashCode()); hashCode = prime * hashCode + ((getExclusiveStartKey() == null) ? 0 : getExclusiveStartKey().hashCode()); hashCode = prime * hashCode + ((getReturnConsumedCapacity() == null) ? 0 : getReturnConsumedCapacity().hashCode()); hashCode = prime * hashCode + ((getTotalSegments() == null) ? 0 : getTotalSegments().hashCode()); hashCode = prime * hashCode + ((getSegment() == null) ? 0 : getSegment().hashCode()); hashCode = prime * hashCode + ((getProjectionExpression() == null) ? 0 : getProjectionExpression().hashCode()); hashCode = prime * hashCode + ((getFilterExpression() == null) ? 0 : getFilterExpression().hashCode()); hashCode = prime * hashCode + ((getExpressionAttributeNames() == null) ? 0 : getExpressionAttributeNames().hashCode()); hashCode = prime * hashCode + ((getExpressionAttributeValues() == null) ? 0 : getExpressionAttributeValues().hashCode()); hashCode = prime * hashCode + ((getConsistentRead() == null) ? 0 : getConsistentRead().hashCode()); return hashCode; }
@Override public Collection<QueuedJob> readAll() { List<QueuedJob> queuedJobs = new LinkedList<>(); Map<String, AttributeValue> lastKey = null; do { LOG.debug("Fetching a page of jobs from DynamoDB!"); ScanRequest scanRequest = new ScanRequest() .withTableName(tableName) .withExclusiveStartKey(lastKey); ScanResult result = client.scan(scanRequest); for (Map<String, AttributeValue> item : result.getItems()) { queuedJobs.add(queuedJobFromMap(item)); } lastKey = result.getLastEvaluatedKey(); } while(lastKey != null); return queuedJobs; }
private List<Document> getNextScanResultSet() { final List<Document> returnValue = new ArrayList<Document>(); final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withAttributesToGet(attributesToGet) .withLimit(limit) .withTableName(tableName) .withConsistentRead(isConsistentRead); request.withSelect(select); request.withScanFilter(this.filter.toConditions()); request.withIndexName(indexName); if (request.getScanFilter() != null && request.getScanFilter().size() > 1) { request.setConditionalOperator(this.conditionalOperator); request.withTotalSegments(totalSegments) .withSegment(segment); final ScanResult result = table.getClient().scan(request); for (final Map<String, AttributeValue> item : result.getItems()) { final Document doc = Document.fromAttributeMap(item); returnValue.add(doc); nextKey = result.getLastEvaluatedKey(); if (nextKey == null || nextKey.size() == 0) { isDone = true;
ScanRequest scanRequest = new ScanRequest(). withTableName(getTableNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); scanRequest = scanRequest.withExclusiveStartKey(Collections. singletonMap(Config._KEY, new AttributeValue(pager.getLastKey()))); ScanResult result = getClient().scan(scanRequest); LinkedList<P> results = new LinkedList<>(); for (Map<String, AttributeValue> item : result.getItems()) { P obj = fromRow(item); if (obj != null) { if (result.getLastEvaluatedKey() != null) { pager.setLastKey(result.getLastEvaluatedKey().get(Config._KEY).getS()); } else if (!results.isEmpty()) {
@Override public int count(final Credentials credentials, final String table, final Map<String, Condition> conditions) { final AmazonDynamoDB aws = credentials.aws(); try { final ScanRequest request = new ScanRequest() .withTableName(table) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withSelect(Select.COUNT) .withLimit(Integer.MAX_VALUE); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); final int count = result.getCount(); Logger.info( this, // @checkstyle LineLength (1 line) "#total(): COUNT=%d in '%s' using %s, %s, in %[ms]s", count, request.getTableName(), request.getFilterExpression(), new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return count; } finally { aws.shutdown(); } }
/** * Generate a list of attribute names found in the Aggregator's dynamo * table. Assumes that all Items in the Aggregator table are of the same * structure. * * @param dynamoClient Dynamo DB Client to use for connection to Dynamo DB. * @param dynamoTable The Dynamo Table for the Aggregator * @return A list of attribute names from the Dynamo table * @throws Exception */ protected List<String> getDictionaryEntry() throws Exception { // get a list of all columns in the table, with keys first List<String> columns = new ArrayList<>(); List<KeySchemaElement> keys = dynamoClient.describeTable(this.tableName).getTable().getKeySchema(); for (KeySchemaElement key : keys) { columns.add(key.getAttributeName()); } ScanResult scan = dynamoClient.scan(new ScanRequest().withTableName(this.tableName).withSelect( Select.ALL_ATTRIBUTES).withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) { for (String s : map.keySet()) { if (!columns.contains(s)) columns.add(s); } } return columns; }
@Override public void run() { ScanRequest scanRequest = new ScanRequest() .withTableName(this.tableName) .withAttributesToGet(this.hashKey) .withSegment(this.workerInstance) .withTotalSegments(threads); Map<String, Set<String>> deduplicated = new HashMap<>(); Set<String> rangeValues = null; scanRequest.withAttributesToGet(this.rangeKey); scanRequest.withLimit(limit); result = dynamoClient.scan(scanRequest .withExclusiveStartKey(lastKeyEvaluated)); if (result.getItems().size() > 0) { returnedResults = true; } else { for (Map<String, AttributeValue> map : result.getItems()) { resultsProcessed++; && result.getLastEvaluatedKey() != null) {
ScanRequest scanRequest = new ScanRequest(). withTableName(getTablNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); scanRequest.setExclusiveStartKey(Collections.singletonMap(Config._KEY, new AttributeValue(pager.getLastKey()))); ScanResult result = client().scan(scanRequest); logger.debug("readPage() CC: {}", result.getConsumedCapacity()); for (Map<String, AttributeValue> item : result.getItems()) { P obj = fromRow(item); if (obj != null) { if (result.getCount() > 0 && results.isEmpty() && result.getLastEvaluatedKey() != null) { pager.setLastKey(result.getLastEvaluatedKey().get(Config._KEY).getS()); return readPage(appid, pager);
switch (searchType) { case SCAN: { final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withTableName(tableName) .withScanFilter(this.filter.toConditions()) .withSelect(Select.COUNT) .withConsistentRead(isConsistentRead); request.withIndexName(indexName); if (request.getScanFilter() != null && request.getScanFilter().size() > 1) { request.setConditionalOperator(this.conditionalOperator); request.withTotalSegments(totalSegments) .withSegment(segment); final ScanResult result = table.getClient().scan(request); count = this.matches.size() + result.getCount(); return count; final QueryResult result = table.getClient().query(request); count = this.matches.size() + result.getCount(); return count;
@Override public <T> ScanResultPage<T> scanPage(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); ScanResultPage<T> result = new ScanResultPage<T>(); List<AttributeTransformer.Parameters<T>> parameters = toParameters(scanResult.getItems(), clazz, scanRequest.getTableName(), config); result.setResults(marshallIntoObjects(parameters)); result.setLastEvaluatedKey(scanResult.getLastEvaluatedKey()); result.setCount(scanResult.getCount()); result.setScannedCount(scanResult.getScannedCount()); result.setConsumedCapacity(scanResult.getConsumedCapacity()); return result; }
final ScanRequest rqst = this.request.withExclusiveStartKey( this.result.getLastEvaluatedKey() ); final long start = System.currentTimeMillis(); final ScanResult rslt = aws.scan(rqst); Logger.info( this, rslt.getCount(), rqst.getTableName(), rqst.getScanFilter(), rslt.getLastEvaluatedKey(), new PrintableConsumedCapacity( rslt.getConsumedCapacity() return new ScanValve.NextDosage(this.credentials, rqst, rslt); } finally { aws.shutdown();
@Test public void scanNextPageOfSegmentWithoutExclusiveStartKey() { EasyMock.reset(mockClient); ScanRequest scan1 = new ScanRequest().withTableName(TABLE); List<ScanRequest> list = new ArrayList<ScanRequest>(); list.add(scan1); Map<String, AttributeValue> lastEvaluatedKey = new HashMap<String, AttributeValue>(); lastEvaluatedKey.put("TestPrimaryKey", new AttributeValue().withS("Test1")); ScanResult result = new ScanResult().withLastEvaluatedKey(lastEvaluatedKey); ParallelScanTask testTask = new ParallelScanTask(mockClient, list); EasyMock.expect(mockClient.scan(scan1)).andReturn(result); EasyMock.replay(mockClient); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); EasyMock.reset(mockClient); Capture<ScanRequest> captureArgument = new Capture<ScanRequest>(); ScanResult mockResult = EasyMock.createMock(ScanResult.class); EasyMock.expect(mockResult.getLastEvaluatedKey()).andReturn(null).anyTimes(); EasyMock.expect(mockClient.scan(EasyMock.capture(captureArgument))).andReturn( mockResult); EasyMock.replay(mockClient, mockResult); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); assertNull(captureArgument.getValue().getExclusiveStartKey()); }
try { while(true) { ScanRequest scanRequest = new ScanRequest() .withTableName(tableName) .withLimit(itemLimit) .withExclusiveStartKey(exclusiveStartKey) .withTotalSegments(totalSegments) .withSegment(segment); ScanResult result = client.scan(scanRequest); totalScannedItemCount += result.getScannedCount(); exclusiveStartKey = result.getLastEvaluatedKey(); if (exclusiveStartKey == null) { break;