/** @deprecated Use {@link TestPipeline} with the {@code DirectRunner}. */ @Deprecated public List<OutputT> processBundle(Iterable<? extends InputT> inputElements) throws Exception { startBundle(); for (InputT inputElement : inputElements) { processElement(inputElement); } finishBundle(); return takeOutputElements(); }
void startElement(WindowedValue<KV<InputT, RestrictionT>> windowedValue) throws Exception { tester.processElement( KeyedWorkItems.elementsWorkItem( "key".getBytes(StandardCharsets.UTF_8), Collections.singletonList(windowedValue))); }
@Test public void fnWithSideInputDefault() throws Exception { PCollection<Integer> pCollection = p.apply(Create.empty(VarIntCoder.of())); final PCollectionView<Integer> value = pCollection.apply(View.<Integer>asSingleton().withDefaultValue(0)); try (DoFnTester<Integer, Integer> tester = DoFnTester.of(new SideInputDoFn(value))) { tester.processElement(1); tester.processElement(2); tester.processElement(4); tester.processElement(8); assertThat(tester.peekOutputElements(), containsInAnyOrder(0, 0, 0, 0)); } }
/** * Advances processing time by a given duration and, if any timers fired, performs a non-seed * {@link DoFn.ProcessElement} call, feeding it the timers. */ boolean advanceProcessingTimeBy(Duration duration) throws Exception { currentProcessingTime = currentProcessingTime.plus(duration); timerInternals.advanceProcessingTime(currentProcessingTime); List<TimerInternals.TimerData> timers = new ArrayList<>(); TimerInternals.TimerData nextTimer; while ((nextTimer = timerInternals.removeNextProcessingTimer()) != null) { timers.add(nextTimer); } if (timers.isEmpty()) { return false; } tester.processElement( KeyedWorkItems.timersWorkItem("key".getBytes(StandardCharsets.UTF_8), timers)); return true; }
@Test public void fnWithSideInputExplicit() throws Exception { PCollection<Integer> pCollection = p.apply(Create.of(-2)); final PCollectionView<Integer> value = pCollection.apply(View.<Integer>asSingleton().withDefaultValue(0)); try (DoFnTester<Integer, Integer> tester = DoFnTester.of(new SideInputDoFn(value))) { tester.setSideInput(value, GlobalWindow.INSTANCE, -2); tester.processElement(16); tester.processElement(32); tester.processElement(64); tester.processElement(128); tester.finishBundle(); assertThat(tester.peekOutputElements(), containsInAnyOrder(-2, -2, -2, -2)); } }
tester = DoFnTester.of(writePartition); tester.setSideInput(tempFilePrefixView, GlobalWindow.INSTANCE, tempFilePrefix); tester.processElement(files);
tester.processElement(1L); tester.processElement(2L); tester.processElement(3L); tester.processElement(4L); tester.processElement(5L); tester.processElement(6L);
@Test public void processElement() throws Exception { for (DoFnTester.CloningBehavior cloning : DoFnTester.CloningBehavior.values()) { try (DoFnTester<Long, String> tester = DoFnTester.of(new CounterDoFn())) { tester.setCloningBehavior(cloning); tester.processElement(1L); List<String> take = tester.takeOutputElements(); assertThat(take, hasItems("1")); // Following takeOutputElements(), neither takeOutputElements() // nor peekOutputElements() return anything. assertTrue(tester.takeOutputElements().isEmpty()); assertTrue(tester.peekOutputElements().isEmpty()); } } }
@Test public void processElementWithOutputTimestamp() throws Exception { try (DoFnTester<Long, String> tester = DoFnTester.of(new CounterDoFn())) { tester.processElement(1L); tester.processElement(2L); List<TimestampedValue<String>> peek = tester.peekOutputElementsWithTimestamp(); TimestampedValue<String> one = TimestampedValue.of("1", new Instant(1000L)); TimestampedValue<String> two = TimestampedValue.of("2", new Instant(2000L)); assertThat(peek, hasItems(one, two)); tester.processElement(3L); tester.processElement(4L); TimestampedValue<String> three = TimestampedValue.of("3", new Instant(3000L)); TimestampedValue<String> four = TimestampedValue.of("4", new Instant(4000L)); peek = tester.peekOutputElementsWithTimestamp(); assertThat(peek, hasItems(one, two, three, four)); List<TimestampedValue<String>> take = tester.takeOutputElementsWithTimestamp(); assertThat(take, hasItems(one, two, three, four)); // Following takeOutputElementsWithTimestamp(), neither takeOutputElementsWithTimestamp() // nor peekOutputElementsWithTimestamp() return anything. assertTrue(tester.takeOutputElementsWithTimestamp().isEmpty()); assertTrue(tester.peekOutputElementsWithTimestamp().isEmpty()); // peekOutputElements() and takeOutputElements() also return nothing. assertTrue(tester.peekOutputElements().isEmpty()); assertTrue(tester.takeOutputElements().isEmpty()); } }
long batchInserted = 0; for (String document : input) { fnTester.processElement(document); numDocsProcessed++; sizeProcessed += document.getBytes(StandardCharsets.UTF_8).length;
tester.processElement(tempTablesElement); tester.finishBundle();
@Test public void peekValuesInWindow() throws Exception { try (DoFnTester<Long, String> tester = DoFnTester.of(new CounterDoFn())) { tester.startBundle(); tester.processElement(1L); tester.processElement(2L); tester.finishBundle(); assertThat( tester.peekOutputElementsInWindow(GlobalWindow.INSTANCE), containsInAnyOrder( TimestampedValue.of("1", new Instant(1000L)), TimestampedValue.of("2", new Instant(2000L)))); assertThat( tester.peekOutputElementsInWindow(new IntervalWindow(new Instant(0L), new Instant(10L))), Matchers.emptyIterable()); } }
long numDocsInserted = 0; for (String document : input) { fnTester.processElement(document); numDocsProcessed++;
long numDocsInserted = 0; for (SolrInputDocument document : input) { fnTester.processElement(document); numDocsProcessed++;