private ScanResult scanNextPageOfSegment(int currentSegment, boolean checkLastEvaluatedKey) { ScanRequest segmentScanRequest = parallelScanRequests.get(currentSegment); if (checkLastEvaluatedKey) { ScanResult lastScanResult = segmentScanResults.get(currentSegment); segmentScanRequest.setExclusiveStartKey(lastScanResult.getLastEvaluatedKey()); } else { segmentScanRequest.setExclusiveStartKey(null); } ScanResult scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(segmentScanRequest)); /** * Cache the scan result in segmentScanResults. * We should never try to get these scan results by calling get() on the cached future tasks. */ segmentScanResults.set(currentSegment, scanResult); /** * Update the state and notify any waiting thread. */ synchronized(segmentScanStates) { if (null == scanResult.getLastEvaluatedKey()) segmentScanStates.set(currentSegment, SegmentScanState.SegmentScanCompleted); else segmentScanStates.set(currentSegment, SegmentScanState.HasNextPage); segmentScanStates.notifyAll(); } return scanResult; }
@Override public Page<Item, ScanOutcome> nextPage() { if (lastEvaluatedKey == null) { throw new NoSuchElementException("No more pages"); } final Integer max = spec.getMaxResultSize(); if (max != null) { int nextLimit = nextRequestLimit(max.intValue()); if (nextLimit == 0) throw new NoSuchElementException("No more pages"); request.setLimit(nextLimit); } request.setExclusiveStartKey(lastEvaluatedKey); // fire off request to the server side ScanResult result = client.scan(request); final int nextIndex = index + this.size(); return new ScanPage(client, spec, request, nextIndex, new ScanOutcome(result)); } }
@Override public <T> PaginatedScanList<T> scan(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); return new PaginatedScanList<T>(this, clazz, db, scanRequest, scanResult, config.getPaginationLoadingStrategy(), config); }
@Override protected synchronized List<T> fetchNextPage() { scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(scanRequest)); return mapper.marshallIntoObjects(mapper.toParameters( scanResult.getItems(), clazz, scanRequest.getTableName(), config)); }
@Override public int count(Class<?> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); scanRequest.setSelect(Select.COUNT); // Count scans can also be truncated for large datasets int count = 0; ScanResult scanResult = null; do { scanResult = db.scan(applyUserAgent(scanRequest)); count += scanResult.getCount(); scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); } while (scanResult.getLastEvaluatedKey() != null); return count; }
@Override public Page<Item, ScanOutcome> firstPage() { ScanRequest request = spec.getRequest(); request.setExclusiveStartKey(startKey); request.setLimit(InternalUtils.minimum( spec.getMaxResultSize(), spec.getMaxPageSize())); ScanResult result = client.scan(request); ScanOutcome outcome = new ScanOutcome(result); setLastLowLevelResult(outcome); return new ScanPage(client, spec, request, 0, outcome); }
@Override public <T> ScanResultPage<T> scanPage(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); ScanResultPage<T> result = new ScanResultPage<T>(); List<AttributeTransformer.Parameters<T>> parameters = toParameters(scanResult.getItems(), clazz, scanRequest.getTableName(), config); result.setResults(marshallIntoObjects(parameters)); result.setLastEvaluatedKey(scanResult.getLastEvaluatedKey()); result.setCount(scanResult.getCount()); result.setScannedCount(scanResult.getScannedCount()); result.setConsumedCapacity(scanResult.getConsumedCapacity()); return result; }
ScanResult scanNextPageOfSegment(int currentSegment, boolean checkLastEvaluatedKey) { final ScanRequest segmentScanRequest = parallelScanRequests.get(currentSegment); if (checkLastEvaluatedKey) { final ScanResult lastScanResult = segmentScanResults.get(currentSegment); segmentScanRequest.setExclusiveStartKey(lastScanResult.getLastEvaluatedKey()); } else { segmentScanRequest.setExclusiveStartKey(null); } final ScanResult scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(segmentScanRequest)); /** * Cache the scan result in segmentScanResults. We should never try to * get these scan results by calling get() on the cached future tasks. */ segmentScanResults.set(currentSegment, scanResult); /** * Update the state and notify any waiting thread. */ synchronized (segmentScanStates) { if (null == scanResult.getLastEvaluatedKey()) { segmentScanStates.set(currentSegment, SegmentScanState.SegmentScanCompleted); } else { segmentScanStates.set(currentSegment, SegmentScanState.HasNextPage); } segmentScanStates.notifyAll(); } return scanResult; }
@Override protected synchronized List<T> fetchNextPage() { scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(scanRequest)); return mapper.marshallIntoObjects(mapper.toParameters( scanResult.getItems(), clazz, scanRequest.getTableName(), config)); } }
@Test public void testIsAllSegmentScansFinished() { ScanRequest scan1 = new ScanRequest().withTableName(TABLE); List<ScanRequest> list = new ArrayList<ScanRequest>(); list.add(scan1); ScanResult resultNoMoreData = new ScanResult().withLastEvaluatedKey(null); ParallelScanTask testTask = new ParallelScanTask(mockClient, list); EasyMock.expect(mockClient.scan(scan1)).andReturn(resultNoMoreData); EasyMock.replay(mockClient); List<ScanResult> firstBatch = testTask.getNextBatchOfScanResults(); assertTrue(testTask.isAllSegmentScanFinished()); } }
final ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); return new PaginatedScanList<T>(this, clazz, db, scanRequest, scanResult, config.getPaginationLoadingStrategy(), config);
@Test public void scanNextPageOfSegmentWithoutExclusiveStartKey() { EasyMock.reset(mockClient); ScanRequest scan1 = new ScanRequest().withTableName(TABLE); List<ScanRequest> list = new ArrayList<ScanRequest>(); list.add(scan1); Map<String, AttributeValue> lastEvaluatedKey = new HashMap<String, AttributeValue>(); lastEvaluatedKey.put("TestPrimaryKey", new AttributeValue().withS("Test1")); ScanResult result = new ScanResult().withLastEvaluatedKey(lastEvaluatedKey); ParallelScanTask testTask = new ParallelScanTask(mockClient, list); EasyMock.expect(mockClient.scan(scan1)).andReturn(result); EasyMock.replay(mockClient); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); EasyMock.reset(mockClient); Capture<ScanRequest> captureArgument = new Capture<ScanRequest>(); ScanResult mockResult = EasyMock.createMock(ScanResult.class); EasyMock.expect(mockResult.getLastEvaluatedKey()).andReturn(null).anyTimes(); EasyMock.expect(mockClient.scan(EasyMock.capture(captureArgument))).andReturn( mockResult); EasyMock.replay(mockClient, mockResult); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); assertNull(captureArgument.getValue().getExclusiveStartKey()); }
@Test public void scanNextPageOfSegmentWithExclusiveStartKey() { EasyMock.reset(mockClient); ScanRequest scan1 = new ScanRequest().withTableName(TABLE); List<ScanRequest> list = new ArrayList<ScanRequest>(); list.add(scan1); Map<String, AttributeValue> lastEvaluatedKey = new HashMap<String, AttributeValue>(); lastEvaluatedKey.put("TestPrimaryKey", new AttributeValue().withS("Test1")); ScanResult result = new ScanResult().withLastEvaluatedKey(lastEvaluatedKey); ParallelScanTask testTask = new ParallelScanTask(mockClient, list); EasyMock.expect(mockClient.scan(scan1)).andReturn(result); EasyMock.replay(mockClient); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); EasyMock.reset(mockClient); Capture<ScanRequest> captureArgument = new Capture<ScanRequest>(); ScanResult mockResult = EasyMock.createMock(ScanResult.class); EasyMock.expect(mockResult.getLastEvaluatedKey()).andReturn(null).anyTimes(); EasyMock.expect(mockClient.scan(EasyMock.capture(captureArgument))).andReturn(mockResult); EasyMock.replay(mockClient, mockResult); testTask.scanNextPageOfSegment(0, true); EasyMock.verify(mockClient); assertEquals( captureArgument.getValue().getExclusiveStartKey().get("TestPrimaryKey").getS(), "Test1"); }
/** * Evaluates the specified scan expression and returns the count of matching * items, without returning any of the actual item data. * <p> * This operation will scan your entire table, and can therefore be very * expensive. Use with caution. * * @param clazz The class mapped to a DynamoDB table. * @param scanExpression The parameters for running the scan. * @param config The configuration to use for this scan, which overrides the * default provided at object construction. * @return The count of matching items, without returning any of the actual * item data. */ public int count(Class<?> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); final ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); scanRequest.setSelect(Select.COUNT); // Count scans can also be truncated for large datasets int count = 0; ScanResult scanResult = null; do { scanResult = db.scan(applyUserAgent(scanRequest)); count += scanResult.getCount(); scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); } while (scanResult.getLastEvaluatedKey() != null); return count; }
final ScanResult result = table.getClient().scan(request); for (final Map<String, AttributeValue> item : result.getItems()) { final Document doc = Document.fromAttributeMap(item);
/** * Scans through an Amazon DynamoDB table and returns a single page of * matching results. The table to scan is determined by looking at the * annotations on the specified class, which declares where to store the * object data in AWS DynamoDB, and the scan expression parameter allows the * caller to filter results and control how the scan is executed. * * @param <T> The type of the objects being returned. * @param clazz The class annotated with DynamoDB annotations describing how * to store the object data in Amazon DynamoDB. * @param scanExpression Details on how to run the scan, including any * filters to apply to limit results. * @param config The configuration to use for this scan, which overrides the * default provided at object construction. * @return a single page of matching results. */ public <T> ScanResultPage<T> scanPage(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); final ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); final ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); final ScanResultPage<T> result = new ScanResultPage<T>(); final List<AttributeTransformer.Parameters<T>> parameters = toParameters(scanResult.getItems(), clazz, scanRequest.getTableName(), config); result.setResults(marshallIntoObjects(parameters)); result.setLastEvaluatedKey(scanResult.getLastEvaluatedKey()); return result; }
final ScanResult result = table.getClient().scan(request); count = this.matches.size() + result.getCount(); return count;
@Override public <T> PaginatedScanList<T> scan(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); return new PaginatedScanList<T>(this, clazz, db, scanRequest, scanResult, config.getPaginationLoadingStrategy(), config); }
@Override protected synchronized List<T> fetchNextPage() { scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(scanRequest)); return mapper.marshallIntoObjects(mapper.toParameters( scanResult.getItems(), clazz, scanRequest.getTableName(), config)); }