Refine search
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getItems() == null) ? 0 : getItems().hashCode()); hashCode = prime * hashCode + ((getCount() == null) ? 0 : getCount().hashCode()); hashCode = prime * hashCode + ((getScannedCount() == null) ? 0 : getScannedCount().hashCode()); hashCode = prime * hashCode + ((getLastEvaluatedKey() == null) ? 0 : getLastEvaluatedKey().hashCode()); hashCode = prime * hashCode + ((getConsumedCapacity() == null) ? 0 : getConsumedCapacity().hashCode()); return hashCode; }
public ScanResult unmarshall(JsonUnmarshallerContext context) throws Exception { ScanResult scanResult = new ScanResult(); if (context.testExpression("Items", targetDepth)) { context.nextToken(); scanResult.setItems(new ListUnmarshaller<java.util.Map<String, AttributeValue>>(new MapUnmarshaller<String, AttributeValue>(context .getUnmarshaller(String.class), AttributeValueJsonUnmarshaller.getInstance())).unmarshall(context)); scanResult.setCount(context.getUnmarshaller(Integer.class).unmarshall(context)); scanResult.setScannedCount(context.getUnmarshaller(Integer.class).unmarshall(context)); scanResult.setLastEvaluatedKey(new MapUnmarshaller<String, AttributeValue>(context.getUnmarshaller(String.class), AttributeValueJsonUnmarshaller.getInstance()).unmarshall(context)); scanResult.setConsumedCapacity(ConsumedCapacityJsonUnmarshaller.getInstance().unmarshall(context));
@Override protected synchronized List<T> fetchNextPage() { scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(scanRequest)); return mapper.marshallIntoObjects(mapper.toParameters( scanResult.getItems(), clazz, scanRequest.getTableName(), config)); }
@Override public int count(Class<?> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); scanRequest.setSelect(Select.COUNT); // Count scans can also be truncated for large datasets int count = 0; ScanResult scanResult = null; do { scanResult = db.scan(applyUserAgent(scanRequest)); count += scanResult.getCount(); scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); } while (scanResult.getLastEvaluatedKey() != null); return count; }
); attrs.addAll(keys); final ScanRequest request = new ScanRequest() .withTableName(table) .withAttributesToGet(attrs) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withLimit(this.limit); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); Logger.info( this, result.getCount(), table, result.getLastEvaluatedKey(), conditions, new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); } finally { aws.shutdown();
ScanRequest scanRequest = new ScanRequest(). withTableName(getTableNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); ScanResult result = getClient().scan(scanRequest); LinkedList<P> results = new LinkedList<>(); for (Map<String, AttributeValue> item : result.getItems()) { P obj = fromRow(item); if (obj != null) { if (result.getLastEvaluatedKey() != null) { pager.setLastKey(result.getLastEvaluatedKey().get(Config._KEY).getS()); } else if (!results.isEmpty()) {
@Override public int count(final Credentials credentials, final String table, final Map<String, Condition> conditions) { final AmazonDynamoDB aws = credentials.aws(); try { final ScanRequest request = new ScanRequest() .withTableName(table) .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL) .withScanFilter(conditions) .withSelect(Select.COUNT) .withLimit(Integer.MAX_VALUE); final long start = System.currentTimeMillis(); final ScanResult result = aws.scan(request); final int count = result.getCount(); Logger.info( this, // @checkstyle LineLength (1 line) "#total(): COUNT=%d in '%s' using %s, %s, in %[ms]s", count, request.getTableName(), request.getFilterExpression(), new PrintableConsumedCapacity( result.getConsumedCapacity() ).print(), System.currentTimeMillis() - start ); return count; } finally { aws.shutdown(); } }
private List<Document> getNextScanResultSet() { final List<Document> returnValue = new ArrayList<Document>(); final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withAttributesToGet(attributesToGet) .withLimit(limit) .withTableName(tableName) final ScanResult result = table.getClient().scan(request); for (final Map<String, AttributeValue> item : result.getItems()) { final Document doc = Document.fromAttributeMap(item); returnValue.add(doc); nextKey = result.getLastEvaluatedKey(); if (nextKey == null || nextKey.size() == 0) { isDone = true;
@Override public Collection<QueuedJob> readAll() { List<QueuedJob> queuedJobs = new LinkedList<>(); Map<String, AttributeValue> lastKey = null; do { LOG.debug("Fetching a page of jobs from DynamoDB!"); ScanRequest scanRequest = new ScanRequest() .withTableName(tableName) .withExclusiveStartKey(lastKey); ScanResult result = client.scan(scanRequest); for (Map<String, AttributeValue> item : result.getItems()) { queuedJobs.add(queuedJobFromMap(item)); } lastKey = result.getLastEvaluatedKey(); } while(lastKey != null); return queuedJobs; }
/** * Generate a list of attribute names found in the Aggregator's dynamo * table. Assumes that all Items in the Aggregator table are of the same * structure. * * @param dynamoClient Dynamo DB Client to use for connection to Dynamo DB. * @param dynamoTable The Dynamo Table for the Aggregator * @return A list of attribute names from the Dynamo table * @throws Exception */ protected List<String> getDictionaryEntry() throws Exception { // get a list of all columns in the table, with keys first List<String> columns = new ArrayList<>(); List<KeySchemaElement> keys = dynamoClient.describeTable(this.tableName).getTable().getKeySchema(); for (KeySchemaElement key : keys) { columns.add(key.getAttributeName()); } ScanResult scan = dynamoClient.scan(new ScanRequest().withTableName(this.tableName).withSelect( Select.ALL_ATTRIBUTES).withLimit(1)); List<Map<String, AttributeValue>> scannedItems = scan.getItems(); for (Map<String, AttributeValue> map : scannedItems) { for (String s : map.keySet()) { if (!columns.contains(s)) columns.add(s); } } return columns; }
credentials); if (region != null) dynamoClient.setRegion(region); final AmazonDynamoDBAsyncClient deleteCli = new AmazonDynamoDBAsyncClient( credentials); .describeTable(dynamoTable).getTable().getKeySchema(); for (KeySchemaElement element : keySchema) { if (element.getKeyType().equals(KeyType.HASH.name())) ScanRequest scan = new ScanRequest() .withTableName(dynamoTable) .withAttributesToGet(keyColumn, dateColumn, StreamAggregator.LAST_WRITE_SEQ) .withExclusiveStartKey(lastKey); ScanResult results = dynamoClient.scan(scan); for (Map<String, AttributeValue> map : results.getItems()) { deleteKey = new HashMap<>(); deleteKey.put(keyColumn, map.get(keyColumn)); lastKey = results.getLastEvaluatedKey(); } while (lastKey != null);
@Override public void run() { ScanRequest scanRequest = new ScanRequest() .withTableName(this.tableName) .withAttributesToGet(this.hashKey) .withSegment(this.workerInstance) .withTotalSegments(threads); scanRequest.withLimit(limit); result = dynamoClient.scan(scanRequest .withExclusiveStartKey(lastKeyEvaluated)); if (result.getItems().size() > 0) { returnedResults = true; } else { for (Map<String, AttributeValue> map : result.getItems()) { resultsProcessed++; && result.getLastEvaluatedKey() != null) { .withS("4000-01-01 00:00:00")); } else { lastKeyEvaluated = result.getLastEvaluatedKey();
ScanRequest req = new ScanRequest(table); req.setAttributesToGet(fields); while (count < recordcount) { req.setExclusiveStartKey(startKey); req.setLimit(recordcount - count); ScanResult res; count += res.getCount(); for (Map<String, AttributeValue> items : res.getItems()) { result.add(extractResult(items)); startKey = res.getLastEvaluatedKey();
@Override public <T> ScanResultPage<T> scanPage(Class<T> clazz, DynamoDBScanExpression scanExpression, DynamoDBMapperConfig config) { config = mergeConfig(config); ScanRequest scanRequest = createScanRequestFromExpression(clazz, scanExpression, config); ScanResult scanResult = db.scan(applyUserAgent(scanRequest)); ScanResultPage<T> result = new ScanResultPage<T>(); List<AttributeTransformer.Parameters<T>> parameters = toParameters(scanResult.getItems(), clazz, scanRequest.getTableName(), config); result.setResults(marshallIntoObjects(parameters)); result.setLastEvaluatedKey(scanResult.getLastEvaluatedKey()); result.setCount(scanResult.getCount()); result.setScannedCount(scanResult.getScannedCount()); result.setConsumedCapacity(scanResult.getConsumedCapacity()); return result; }
switch (searchType) { case SCAN: { final ScanRequest request = new ScanRequest(); request.withExclusiveStartKey(nextKey) .withTableName(tableName) .withScanFilter(this.filter.toConditions()) .withSelect(Select.COUNT) final ScanResult result = table.getClient().scan(request); count = this.matches.size() + result.getCount(); return count; final QueryResult result = table.getClient().query(request); count = this.matches.size() + result.getCount(); return count;
ScanRequest scanRequest = new ScanRequest(). withTableName(getTablNameForAppid(appid)). withLimit(pager.getLimit()). withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL); logger.debug("readPage() CC: {}", result.getConsumedCapacity()); for (Map<String, AttributeValue> item : result.getItems()) { P obj = fromRow(item); if (obj != null) { if (result.getCount() > 0 && results.isEmpty() && result.getLastEvaluatedKey() != null) { pager.setLastKey(result.getLastEvaluatedKey().get(Config._KEY).getS()); return readPage(appid, pager);
final ScanRequest rqst = this.request.withExclusiveStartKey( this.result.getLastEvaluatedKey() ); final long start = System.currentTimeMillis(); final ScanResult rslt = aws.scan(rqst); Logger.info( this, rslt.getCount(), rqst.getTableName(), rqst.getScanFilter(), rslt.getLastEvaluatedKey(), new PrintableConsumedCapacity( rslt.getConsumedCapacity() ).print(), System.currentTimeMillis() - start return new ScanValve.NextDosage(this.credentials, rqst, rslt); } finally { aws.shutdown();
@Override public List<String> getRows(String storeName, String continuationToken, int count) { String tableName = storeToTableName(storeName); ScanRequest scanRequest = new ScanRequest(tableName); scanRequest.setAttributesToGet(Arrays.asList(ROW_KEY_ATTR_NAME)); // attributes to get if (continuationToken != null) { scanRequest.setExclusiveStartKey(makeDDBKey(continuationToken)); } List<String> rowKeys = new ArrayList<>(); while (rowKeys.size() < count) { ScanResult scanResult = scan(scanRequest); List<Map<String, AttributeValue>> itemList = scanResult.getItems(); if (itemList.size() == 0) { break; } for (Map<String, AttributeValue> attributeMap : itemList) { AttributeValue rowAttr = attributeMap.get(ROW_KEY_ATTR_NAME); rowKeys.add(rowAttr.getS()); if (rowKeys.size() >= count) { break; } } Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey(); if (lastEvaluatedKey == null) { break; } scanRequest.setExclusiveStartKey(lastEvaluatedKey); } return rowKeys; }
@Test public void scanNextPageOfSegmentWithoutExclusiveStartKey() { EasyMock.reset(mockClient); ScanRequest scan1 = new ScanRequest().withTableName(TABLE); List<ScanRequest> list = new ArrayList<ScanRequest>(); list.add(scan1); Map<String, AttributeValue> lastEvaluatedKey = new HashMap<String, AttributeValue>(); lastEvaluatedKey.put("TestPrimaryKey", new AttributeValue().withS("Test1")); ScanResult result = new ScanResult().withLastEvaluatedKey(lastEvaluatedKey); ParallelScanTask testTask = new ParallelScanTask(mockClient, list); EasyMock.expect(mockClient.scan(scan1)).andReturn(result); EasyMock.replay(mockClient); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); EasyMock.reset(mockClient); Capture<ScanRequest> captureArgument = new Capture<ScanRequest>(); ScanResult mockResult = EasyMock.createMock(ScanResult.class); EasyMock.expect(mockResult.getLastEvaluatedKey()).andReturn(null).anyTimes(); EasyMock.expect(mockClient.scan(EasyMock.capture(captureArgument))).andReturn( mockResult); EasyMock.replay(mockClient, mockResult); testTask.scanNextPageOfSegment(0, false); EasyMock.verify(mockClient); assertNull(captureArgument.getValue().getExclusiveStartKey()); }
@Override protected synchronized List<T> fetchNextPage() { scanRequest.setExclusiveStartKey(scanResult.getLastEvaluatedKey()); scanResult = dynamo.scan(DynamoDBMapper.applyUserAgent(scanRequest)); return mapper.marshallIntoObjects(mapper.toParameters( scanResult.getItems(), clazz, scanRequest.getTableName(), config)); } }