@Test public void testSourceTransform() { PTransform<? super PBegin, ? extends POutput> myTransform = TextIO.read().from("foo.*"); DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(myTransform); assertThat(displayData, hasItem(hasDisplayItem("filePattern", "foo.*"))); } }
@Test @Category({ValidatesRunner.class, UsesUnboundedPCollections.class}) public void testPrimitiveReadDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); Set<DisplayData> displayData; PubsubIO.Read<String> baseRead = PubsubIO.readStrings(); // Reading from a subscription. PubsubIO.Read<String> read = baseRead.fromSubscription("projects/project/subscriptions/subscription"); displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "PubsubIO.Read should include the subscription in its primitive display data", displayData, hasItem(hasDisplayItem("subscription"))); // Reading from a topic. read = baseRead.fromTopic("projects/project/topics/topic"); displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "PubsubIO.Read should include the topic in its primitive display data", displayData, hasItem(hasDisplayItem("topic"))); }
@Test @Category(ValidatesRunner.class) public void testPrimitiveReadDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); AvroIO.Read<GenericRecord> read = AvroIO.readGenericRecords(Schema.create(Schema.Type.STRING)).from("/foo.*"); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "AvroIO.Read should include the file pattern in its primitive transform", displayData, hasItem(hasDisplayItem("filePattern"))); }
@Test @Category(ValidatesRunner.class) public void testPrimitiveReadDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); TextIO.Read read = TextIO.read().from("foobar"); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "TextIO.Read should include the file prefix in its primitive display data", displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar"))))); }
@Test public void testSourcePrimitiveDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); int numSplits = 98; PTransform<PBegin, PCollection<Entity>> read = DatastoreIO.v1() .read() .withProjectId(PROJECT_ID) .withQuery(Query.newBuilder().build()) .withNumQuerySplits(numSplits); String assertMessage = "DatastoreIO read should include the '%s' in its primitive display data"; Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( String.format(assertMessage, "project id"), displayData, hasItem(hasDisplayItem("projectId", PROJECT_ID))); assertThat( String.format(assertMessage, "number of query splits"), displayData, hasItem(hasDisplayItem("numQuerySplits", numSplits))); }
@Test public void testReadingPrimitiveDisplayData() throws IOException, InterruptedException { final String table = "fooTable"; service.createTable(table); RowFilter rowFilter = RowFilter.newBuilder().setRowKeyRegexFilter(ByteString.copyFromUtf8("foo.*")).build(); DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigtableIO.Read read = BigtableIO.read() .withBigtableOptions(BIGTABLE_OPTIONS) .withTableId(table) .withRowFilter(rowFilter) .withBigtableService(service); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "BigtableIO.Read should include the table id in its primitive display data", displayData, Matchers.hasItem(hasDisplayItem("tableId"))); assertThat( "BigtableIO.Read should include the row filter, if it exists, in its primitive " + "display data", displayData, Matchers.hasItem(hasDisplayItem("rowFilter"))); }
@Test public void testTableSourcePrimitiveDisplayData() throws IOException, InterruptedException { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read read = BigQueryIO.read() .from("project:dataset.tableId") .withTestServices( new FakeBigQueryServices() .withDatasetService(new FakeDatasetService()) .withJobService(new FakeJobService())) .withoutValidation(); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "BigQueryIO.Read should include the table spec in its primitive display data", displayData, hasItem(hasDisplayItem("table"))); }
@Test public void testQuerySourcePrimitiveDisplayData() throws IOException, InterruptedException { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read read = BigQueryIO.read() .fromQuery("foobar") .withTestServices( new FakeBigQueryServices() .withDatasetService(new FakeDatasetService()) .withJobService(new FakeJobService())) .withoutValidation(); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "BigQueryIO.Read should include the query in its primitive display data", displayData, hasItem(hasDisplayItem("query"))); }