@java.lang.Override public Builder newBuilderForType() { return newBuilder(); }
/** * * * <pre> * The parsed form of the `GqlQuery` from the request, if it was set. * </pre> * * <code>.google.datastore.v1.Query query = 2;</code> */ public Builder mergeQuery(com.google.datastore.v1.Query value) { if (queryBuilder_ == null) { if (query_ != null) { query_ = com.google.datastore.v1.Query.newBuilder(query_).mergeFrom(value).buildPartial(); } else { query_ = value; } onChanged(); } else { queryBuilder_.mergeFrom(value); } return this; } /**
&& queryType_ != com.google.datastore.v1.Query.getDefaultInstance()) { queryType_ = com.google.datastore.v1.Query.newBuilder((com.google.datastore.v1.Query) queryType_) .mergeFrom(value) .buildPartial();
com.google.datastore.v1.Query toPb() { com.google.datastore.v1.Query.Builder queryPb = com.google.datastore.v1.Query.newBuilder(); if (kind != null) { queryPb.addKindBuilder().setName(kind);
@java.lang.Override public Builder newBuilderForType() { return newBuilder(); }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() {
Query() { query = com.google.datastore.v1.Query.newBuilder(); filters = Lists.newArrayList(); }
/** * <pre> * The query to run. * </pre> * * <code>optional .google.datastore.v1.Query query = 3;</code> */ public Builder mergeQuery(com.google.datastore.v1.Query value) { if (queryBuilder_ == null) { if (queryTypeCase_ == 3 && queryType_ != com.google.datastore.v1.Query.getDefaultInstance()) { queryType_ = com.google.datastore.v1.Query.newBuilder((com.google.datastore.v1.Query) queryType_) .mergeFrom(value).buildPartial(); } else { queryType_ = value; } onChanged(); } else { if (queryTypeCase_ == 3) { queryBuilder_.mergeFrom(value); } queryBuilder_.setMessage(value); } queryTypeCase_ = 3; return this; } /**
/** * List the greetings in the specified guestbook. */ private void listGreetings(String guestbookName) throws DatastoreException { Query.Builder query = Query.newBuilder(); query.addKindBuilder().setName(GREETING_KIND); query.setFilter(makeFilter(KEY_PROPERTY, PropertyFilter.Operator.HAS_ANCESTOR, makeValue(makeKey(GUESTBOOK_KIND, guestbookName)))); query.addOrder(makeOrder(DATE_PROPERTY, PropertyOrder.Direction.DESCENDING)); List<Entity> greetings = runQuery(query.build()); if (greetings.size() == 0) { System.out.println("no greetings in " + guestbookName); } for (Entity greeting : greetings) { Map<String, Value> propertyMap = greeting.getProperties(); System.out.println( DatastoreHelper.toDate(propertyMap.get(DATE_PROPERTY)) + ": " + DatastoreHelper.getString(propertyMap.get(USER_PROPERTY)) + " says " + DatastoreHelper.getString(propertyMap.get(MESSAGE_PROPERTY))); } }
/** Generate dummy query splits. */ private List<Query> splitQuery(Query query, int numSplits) { List<Query> queries = new ArrayList<>(); int offsetOfOriginal = query.getOffset(); for (int i = 0; i < numSplits; i++) { Query.Builder q = Query.newBuilder(); q.addKindBuilder().setName(KIND); // Making sub-queries unique (and not equal to the original query) by setting different // offsets. q.setOffset(++offsetOfOriginal); queries.add(q.build()); } return queries; }
/** Helper function to run a test reading from a {@link ReadFn}. */ private void readFnTest(int numEntities) throws Exception { // An empty query to read entities. Query query = Query.newBuilder().setLimit(Int32Value.newBuilder().setValue(numEntities)).build(); // Use mockResponseForQuery to generate results. when(mockDatastore.runQuery(any(RunQueryRequest.class))) .thenAnswer( invocationOnMock -> { Query q = ((RunQueryRequest) invocationOnMock.getArguments()[0]).getQuery(); return mockResponseForQuery(q); }); ReadFn readFn = new ReadFn(V_1_OPTIONS, mockDatastoreFactory); DoFnTester<Query, Entity> doFnTester = DoFnTester.of(readFn); /** * Although Datastore client is marked transient in {@link ReadFn}, when injected through mock * factory using a when clause for unit testing purposes, it is not serializable because it * doesn't have a no-arg constructor. Thus disabling the cloning to prevent the test object from * being serialized. */ doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE); List<Entity> entities = doFnTester.processBundle(query); int expectedNumCallsToRunQuery = (int) Math.ceil((double) numEntities / QUERY_BATCH_LIMIT); verify(mockDatastore, times(expectedNumCallsToRunQuery)).runQuery(any(RunQueryRequest.class)); // Validate the number of results. assertEquals(numEntities, entities.size()); }
/** A helper method to test {@link SplitQueryFn} to generate the expected number of splits. */ private void testSplitQueryFn( String projectId, String kind, @Nullable String namespace, int expectedNumSplits) throws Exception { Query.Builder query = Query.newBuilder(); query.addKindBuilder().setName(kind); SplitQueryFn splitQueryFn = new SplitQueryFn(V1Options.from(projectId, namespace, null), 0); DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn); List<Query> queries = doFnTester.processBundle(query.build()); assertEquals(expectedNumSplits, queries.size()); }
/** Build a datastore ancestor query for the specified kind, namespace and ancestor. */ static Query makeAncestorKindQuery(String kind, @Nullable String namespace, String ancestor) { Query.Builder q = Query.newBuilder(); q.addKindBuilder().setName(kind); q.setFilter( makeFilter( "__key__", PropertyFilter.Operator.HAS_ANCESTOR, makeValue(makeAncestorKey(namespace, kind, ancestor)))); return q.build(); }
/** Tests that {@link ReadFn} retries after an error. */ @Test public void testReadFnRetriesErrors() throws Exception { // An empty query to read entities. Query query = Query.newBuilder().setLimit(Int32Value.newBuilder().setValue(1)).build(); // Use mockResponseForQuery to generate results. when(mockDatastore.runQuery(any(RunQueryRequest.class))) .thenThrow(new DatastoreException("RunQuery", Code.DEADLINE_EXCEEDED, "", null)) .thenAnswer( invocationOnMock -> { Query q = ((RunQueryRequest) invocationOnMock.getArguments()[0]).getQuery(); return mockResponseForQuery(q); }); ReadFn readFn = new ReadFn(V_1_OPTIONS, mockDatastoreFactory); DoFnTester<Query, Entity> doFnTester = DoFnTester.of(readFn); doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE); doFnTester.processBundle(query); }
/** * Creates a scatter query from the given user query * * @param query the user's query. * @param numSplits the number of splits to create. */ private Query.Builder createScatterQuery(Query query, int numSplits) { // TODO(pcostello): We can potentially support better splits with equality filters in our query // if there exists a composite index on property, __scatter__, __key__. Until an API for // metadata exists, this isn't possible. Note that ancestor and inequality queries fall into // the same category. Query.Builder scatterPointQuery = Query.newBuilder(); scatterPointQuery.addAllKind(query.getKindList()); scatterPointQuery.addOrder(DatastoreHelper.makeOrder( DatastoreHelper.SCATTER_PROPERTY_NAME, Direction.ASCENDING)); // There is a split containing entities before and after each scatter entity: // ||---*------*------*------*------*------*------*---|| = scatter entity // If we represent each split as a region before a scatter entity, there is an extra region // following the last scatter point. Thus, we do not need the scatter entities for the last // region. scatterPointQuery.getLimitBuilder().setValue((numSplits - 1) * KEYS_PER_SPLIT); scatterPointQuery.addProjection(Projection.newBuilder().setProperty( PropertyReference.newBuilder().setName("__key__"))); return scatterPointQuery; }
@Test public void testSourcePrimitiveDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); int numSplits = 98; PTransform<PBegin, PCollection<Entity>> read = DatastoreIO.v1() .read() .withProjectId(PROJECT_ID) .withQuery(Query.newBuilder().build()) .withNumQuerySplits(numSplits); String assertMessage = "DatastoreIO read should include the '%s' in its primitive display data"; Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( String.format(assertMessage, "project id"), displayData, hasItem(hasDisplayItem("projectId", PROJECT_ID))); assertThat( String.format(assertMessage, "number of query splits"), displayData, hasItem(hasDisplayItem("numQuerySplits", numSplits))); }
@Test public void testReadValidationFailsQueryLimitZero() throws Exception { Query invalidLimit = Query.newBuilder().setLimit(Int32Value.newBuilder().setValue(0)).build(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Invalid query limit 0: must be positive"); DatastoreIO.v1().read().withQuery(invalidLimit); }
@Test public void testReadValidationFailsQueryLimitNegative() throws Exception { Query invalidLimit = Query.newBuilder().setLimit(Int32Value.newBuilder().setValue(-5)).build(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Invalid query limit -5: must be positive"); DatastoreIO.v1().read().withQuery(invalidLimit); }
/** Builds a latest timestamp statistics query. */ private static Query makeLatestTimestampQuery(String namespace) { Query.Builder timestampQuery = Query.newBuilder(); if (namespace == null) { timestampQuery.addKindBuilder().setName("__Stat_Total__"); } else { timestampQuery.addKindBuilder().setName("__Stat_Ns_Total__"); } timestampQuery.addOrder(makeOrder("timestamp", DESCENDING)); timestampQuery.setLimit(Int32Value.newBuilder().setValue(1)); return timestampQuery.build(); }
/** Builds a per-kind statistics query for the given timestamp and namespace. */ private static Query makeStatKindQuery(String namespace, long timestamp) { Query.Builder statQuery = Query.newBuilder(); if (namespace == null) { statQuery.addKindBuilder().setName("__Stat_Kind__"); } else { statQuery.addKindBuilder().setName("__Stat_Ns_Kind__"); } statQuery.setFilter( makeAndFilter( makeFilter("kind_name", EQUAL, makeValue(KIND).build()).build(), makeFilter("timestamp", EQUAL, makeValue(timestamp * 1000000L).build()).build())); return statQuery.build(); }