@Override public void processElement(WindowedValue<InputT> compressedElement) throws Exception { for (WindowedValue<InputT> element : compressedElement.explodeWindows()) { Collection<? extends BoundedWindow> windows = assignWindows(windowFn, element); outputBundle.add( WindowedValue.of( element.getValue(), element.getTimestamp(), windows, element.getPane())); } }
/** Extract the windows associated with the values. */ private Set<W> collectWindows(Iterable<WindowedValue<InputT>> values) throws Exception { Set<W> windows = new HashSet<>(); for (WindowedValue<?> value : values) { for (BoundedWindow untypedWindow : value.getWindows()) { @SuppressWarnings("unchecked") W window = (W) untypedWindow; windows.add(window); } } return windows; }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(WindowedValue.valueInGlobalWindow(workItem)); } } }
@Override public int hashCode() { // Hash only the millis of the timestamp to be consistent with equals return Objects.hash(getValue(), getTimestamp().getMillis(), getWindows(), getPane()); }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(in.withValue(workItem)); } } }
@Test public void testExplodeWindowsInOneWindowEquals() { Instant now = Instant.now(); BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L)); WindowedValue<String> value = WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING); assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value)); }
@Override public WindowedValue<T> decode(InputStream inStream, Context context) throws CoderException, IOException { T value = valueCoder.decode(inStream, context); return WindowedValue.valueInGlobalWindow(value); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo( Coder<T> coder, WindowingStrategy<?, ?> windowingStrategy) { WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( coder, windowingStrategy.getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
/** @deprecated for use only in compatibility with old broken code */ @Deprecated static <T> WindowedValue<T> createWithoutValidation( T value, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) { if (windows.size() == 1) { return of(value, timestamp, windows.iterator().next(), pane); } else { return new TimestampedValueInMultipleWindows<>(value, timestamp, windows, pane); } }
@Override public void processElement(WindowedValue<InputT> input) { // StatefulDoFnRunner always observes windows, so we need to explode for (WindowedValue<InputT> value : input.explodeWindows()) { BoundedWindow window = value.getWindows().iterator().next(); if (isLate(window)) { // The element is too late for this window. droppedDueToLateness.inc(); WindowTracing.debug( "StatefulDoFnRunner.processElement: Dropping element at {}; window:{} " + "since too far behind inputWatermark:{}", input.getTimestamp(), window, cleanupTimer.currentInputWatermarkTime()); } else { cleanupTimer.setForWindow(window); doFnRunner.processElement(value); } } }
Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( WindowedValue.of( new Object(), Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)));
@Test @SuppressWarnings("unchecked") public void testSingleOutput() throws Exception { Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); TupleTag<String> outputTag = new TupleTag<>("main-output"); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new IdentityDoFn<>(), "stepName", coder, null, Collections.emptyMap(), outputTag, Collections.emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder), WindowingStrategy.globalDefault(), new HashMap<>(), /* side-input mapping */ Collections.emptyList(), /* side inputs */ PipelineOptionsFactory.as(FlinkPipelineOptions.class), null, null); OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness = new OneInputStreamOperatorTestHarness<>(doFnOperator); testHarness.open(); testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("Hello"))); assertThat( stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains(WindowedValue.valueInGlobalWindow("Hello"))); testHarness.close(); }
@Override public Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) { if (views.isEmpty()) { // When there are no side inputs, we can preserve the compressed representation. underlying.processElement(elem); return Collections.emptyList(); } ImmutableList.Builder<WindowedValue<InputT>> pushedBack = ImmutableList.builder(); for (WindowedValue<InputT> windowElem : elem.explodeWindows()) { BoundedWindow mainInputWindow = Iterables.getOnlyElement(windowElem.getWindows()); if (isReady(mainInputWindow)) { // When there are any side inputs, we have to process the element in each window // individually, to disambiguate access to per-window side inputs. underlying.processElement(windowElem); } else { notReadyWindows.add(mainInputWindow); pushedBack.add(windowElem); } } return pushedBack.build(); }
@SuppressWarnings("ConstantConditions") public JavaRDD<WindowedValue<T>> getRDD() { if (rdd == null) { WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder = WindowedValue.getValueOnlyCoder(coder); rdd = jsc.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder)) .map(CoderHelpers.fromByteFunction(windowCoder)); } return rdd; }
private static Instant minTimestamp(Iterable<? extends WindowedValue<?>> elements) { Instant minTs = BoundedWindow.TIMESTAMP_MAX_VALUE; for (WindowedValue<?> element : elements) { if (element.getTimestamp().isBefore(minTs)) { minTs = element.getTimestamp(); } } return minTs; } }
@Override public void processElement(WindowedValue<InputT> compressedElem) { if (observesWindow) { for (WindowedValue<InputT> elem : compressedElem.explodeWindows()) { invokeProcessElement(elem); } } else { invokeProcessElement(compressedElem); } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(in.withValue(workItem)); } } }
@Test public void testExplodeWindowsManyWindowsMultipleWindowedValues() { Instant now = Instant.now(); BoundedWindow centerWindow = new IntervalWindow(now.minus(1000L), now.plus(1000L)); BoundedWindow pastWindow = new IntervalWindow(now.minus(1500L), now.plus(500L)); BoundedWindow futureWindow = new IntervalWindow(now.minus(500L), now.plus(1500L)); BoundedWindow futureFutureWindow = new IntervalWindow(now, now.plus(2000L)); PaneInfo pane = PaneInfo.createPane(false, false, Timing.ON_TIME, 3L, 0L); WindowedValue<String> value = WindowedValue.of( "foo", now, ImmutableList.of(pastWindow, centerWindow, futureWindow, futureFutureWindow), pane); assertThat( value.explodeWindows(), containsInAnyOrder( WindowedValue.of("foo", now, futureFutureWindow, pane), WindowedValue.of("foo", now, futureWindow, pane), WindowedValue.of("foo", now, centerWindow, pane), WindowedValue.of("foo", now, pastWindow, pane))); } }