private <T> ValueInSingleWindow<T> wrapValue(T value) { return ValueInSingleWindow.of( value, GlobalWindow.TIMESTAMP_MAX_VALUE, GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING); }
@ProcessElement public void processElement( @Element T element, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<ValueInSingleWindow<T>> r) { r.outputWithTimestamp( ValueInSingleWindow.of(element, timestamp, window, pane), timestamp); } }))
public long insertAll(TableReference ref, List<TableRow> rowList) throws IOException, InterruptedException { List<ValueInSingleWindow<TableRow>> rows = rowList.stream() .map(r -> ValueInSingleWindow.of( r, BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)) .collect(Collectors.toList()); return bqServices.getDatasetService(bqOptions) .insertAll(ref, rows, null, InsertRetryPolicy.alwaysRetry(), new ArrayList<>(), ErrorContainer.TABLE_ROW_ERROR_CONTAINER, false, false); }
public long insertAll( TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList) throws IOException, InterruptedException { List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList(); for (TableRow row : rowList) { windowedRows.add( ValueInSingleWindow.of( row, GlobalWindow.TIMESTAMP_MAX_VALUE, GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING)); } return insertAll( ref, windowedRows, insertIdList, InsertRetryPolicy.alwaysRetry(), null, null, false, false); }
@ProcessElement public void processElement( @Element KV<K, V> element, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<KV<K, ValueInSingleWindow<V>>> r) { r.output( KV.of( element.getKey(), ValueInSingleWindow.of(element.getValue(), timestamp, window, pane))); } }))
@Override public <AdditionalOutputT> void outputWindowedValue( TupleTag<AdditionalOutputT> tag, AdditionalOutputT output, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) { for (BoundedWindow window : windows) { tester.getMutableOutput(tag).add(ValueInSingleWindow.of(output, timestamp, window, pane)); } } }
/** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */ @ProcessElement public void processElement(ProcessContext context, BoundedWindow window) { String tableSpec = context.element().getKey().getKey(); List<ValueInSingleWindow<TableRow>> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec); List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows, tableSpec); rows.add( ValueInSingleWindow.of( context.element().getValue().tableRow, context.timestamp(), window, context.pane())); uniqueIds.add(context.element().getValue().uniqueId); }
@Test public void nonLatePanesSingleOnTime() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.nonLatePanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING), ValueInSingleWindow.of( 2, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING)); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(2, 4, 8)); }
@Test public void allPanesSinglePane() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.allPanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING), ValueInSingleWindow.of( 2, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING)); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(2, 4, 8)); }
@Test public void onlyPaneNoFiring() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.onlyPane(PAssert.PAssertionSite.capture("")); Iterable<ValueInSingleWindow<Integer>> noFiring = ImmutableList.of( ValueInSingleWindow.of( 9, BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING), ValueInSingleWindow.of( 19, BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)); assertThat(extractor.apply(noFiring), containsInAnyOrder(9, 19)); }
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) throws IOException { dynamicDestinations.setSideInputAccessorFromProcessContext(context); ValueInSingleWindow<T> element = ValueInSingleWindow.of( context.element(), context.timestamp(), window, context.pane()); DestinationT tableDestination = dynamicDestinations.getDestination(element); checkArgument( tableDestination != null, "DynamicDestinations.getDestination() may not return null, " + "but %s returned null on element %s", dynamicDestinations, element); TableRow tableRow = formatFunction.apply(context.element()); checkArgument( tableRow != null, "formatFunction may not return null, but %s returned null on element %s", formatFunction, context.element()); context.output(KV.of(tableDestination, tableRow)); } })
@Test public void onlyPaneOnlyOneFiring() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.onlyPane(PAssert.PAssertionSite.capture("")); Iterable<ValueInSingleWindow<Integer>> onlyFiring = ImmutableList.of( ValueInSingleWindow.of( 2, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING), ValueInSingleWindow.of( 1, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING)); assertThat(extractor.apply(onlyFiring), containsInAnyOrder(2, 1)); }
@Test public void nonLatePanesMultiplePanes() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.nonLatePanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.LATE, 2L, 1L)), ValueInSingleWindow.of(7, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)), ValueInSingleWindow.of( 1, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(true, false, Timing.EARLY))); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 1, 7)); }
@Test public void onTimePane() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.onTimePane(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)), ValueInSingleWindow.of( 2, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L))); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(2, 4)); }
@Test public void finalPane() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.finalPane(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, true, Timing.LATE, 2L, 1L)), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)), ValueInSingleWindow.of( 1, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(true, false, Timing.EARLY))); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(8)); }
@Test public void allPanesMultiplePanes() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.allPanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.LATE, 2L, 1L)), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.ON_TIME, 1L, 0L)), ValueInSingleWindow.of( 1, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(true, false, Timing.EARLY))); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8, 1)); }
@Test public void nonLatePanesSingleEarly() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.nonLatePanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(true, false, Timing.EARLY)), ValueInSingleWindow.of( 4, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(true, false, Timing.EARLY))); assertThat(extractor.apply(onlyOnTime), containsInAnyOrder(4, 8)); }
@Test public void allPanesSingleLate() { SerializableFunction<Iterable<ValueInSingleWindow<Integer>>, Iterable<Integer>> extractor = PaneExtractors.nonLatePanes(); Iterable<ValueInSingleWindow<Integer>> onlyOnTime = ImmutableList.of( ValueInSingleWindow.of( 8, new Instant(0L), GlobalWindow.INSTANCE, PaneInfo.createPane(false, false, Timing.LATE, 0L, 0L))); assertThat(extractor.apply(onlyOnTime), emptyIterable()); }
@Test public void testDecodeEncodeEqual() throws Exception { Instant now = Instant.now(); ValueInSingleWindow<String> value = ValueInSingleWindow.of( "foo", now, new IntervalWindow(now, now.plus(Duration.standardSeconds(10))), PaneInfo.NO_FIRING); CoderProperties.coderDecodeEncodeEqual( ValueInSingleWindow.Coder.of(StringUtf8Coder.of(), IntervalWindow.getCoder()), value); }
@Test @Category(NeedsRunner.class) public void globalWindowNoKeys() { PCollection<ValueInSingleWindow<String>> result = pipeline .apply( TestStream.create(StringUtf8Coder.of()) .addElements(TimestampedValue.of("dei", new Instant(123L))) .advanceWatermarkToInfinity()) .apply(Reify.windows()); PAssert.that(result) .containsInAnyOrder( ValueInSingleWindow.of( "dei", new Instant(123L), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)); pipeline.run(); }