private Collection<Entity> fetchAndFilter(Query query, List<ColumnCondition<Entity, Object>> locals) { System.err.println(query); PreparedQuery prepared = datastore.prepare(query); List<Entity> list = prepared.asList(FetchOptions.Builder.withChunkSize(CHUNKSIZE)); if (!locals.isEmpty()) { List<Entity> flist = new ArrayList<>(); for (Entity entity : list) { boolean ok = true; for (ColumnCondition cc : locals) { if (cc.matches(converter, entity) != TruthValue.TRUE) { ok = false; break; } } if (ok) { flist.add(entity); } } System.err.println("filtered from "+list.size()+" to "+flist.size()); return flist; } System.err.println("fetched "+list.size()); return list; }
.setAncestor(FormEntity.key(formClass).getRaw())); FetchOptions fetchOptions = FetchOptions.Builder.withChunkSize(500).prefetchSize(500); Iterator<Entity> it = preparedQuery.asIterator(fetchOptions);
@Test public void testChunkSize() { List<Entity> results = executeQuery(withChunkSize(2)); assertEquals(5, results.size()); }
opts = withChunkSize(fetchSize); } else { opts.chunkSize(fetchSize);
private void deleteAll(final String kind, final Key rootJobKey) { logger.info("Deleting all " + kind + " with rootJobKey=" + rootJobKey); final int chunkSize = 100; final FetchOptions fetchOptions = FetchOptions.Builder.withChunkSize(chunkSize); final PreparedQuery preparedQuery = dataStore.prepare(new Query(kind).setKeysOnly().setFilter( new FilterPredicate(ROOT_JOB_KEY_PROPERTY, EQUAL, rootJobKey))); tryFiveTimes(new Operation<Void>("delete") { @Override public Void call() { Iterator<Entity> iter = preparedQuery.asIterator(fetchOptions); while (iter.hasNext()) { ArrayList<Key> keys = new ArrayList<>(chunkSize); for (int i = 0; i < chunkSize && iter.hasNext(); i++) { keys.add(iter.next().getKey()); } logger.info("Deleting " + keys.size() + " " + kind + "s with rootJobKey=" + rootJobKey); dataStore.delete(null, keys); } return null; } }); }
private FetchOptions getFetchOptionsWithoutOffsetAndLimit( FetchOptions opts, Integer chunkSize, Integer prefetchSize) { FetchOptions optsWithoutOffsetAndLimit = null; if (chunkSize != null) { optsWithoutOffsetAndLimit = FetchOptions.Builder.withChunkSize(opts.getChunkSize()); if (prefetchSize != null) { optsWithoutOffsetAndLimit.prefetchSize(opts.getPrefetchSize()); } } else if (prefetchSize != null) { optsWithoutOffsetAndLimit = FetchOptions.Builder.withPrefetchSize(prefetchSize); } return optsWithoutOffsetAndLimit; }
public List<Entity> queryAll(final String kind, final Key rootJobKey) { Query query = new Query(kind); query.setFilter(new FilterPredicate(ROOT_JOB_KEY_PROPERTY, EQUAL, rootJobKey)); final PreparedQuery preparedQuery = dataStore.prepare(query); final FetchOptions options = FetchOptions.Builder.withChunkSize(500); return tryFiveTimes(new Operation<List<Entity>>("queryFullPipeline") { @Override public List<Entity> call() { return preparedQuery.asList(options); } }); }
@Override public List<Entity> getAll(String kind) { Query query = new Query(kind); PreparedQuery prepared = datastore.prepare(query); return prepared.asList(FetchOptions.Builder.withChunkSize(CHUNKSIZE)); }