@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) { WindowedStructuralKey<K> key = WindowedStructuralKey.create(keyCoder, context.element().getKey(), window); AccumT accumulator = accumulators.get(key); Instant assignedTs = timestampCombiner.assign(window, context.timestamp()); if (accumulator == null) { accumulator = combineFn.createAccumulator(); accumulators.put(key, accumulator); timestamps.put(key, assignedTs); } accumulators.put(key, combineFn.addInput(accumulator, context.element().getValue())); timestamps.put(key, timestampCombiner.combine(assignedTs, timestamps.get(key))); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } FlinkWatermarkHoldState<?, ?> that = (FlinkWatermarkHoldState<?, ?>) o; if (!address.equals(that.address)) { return false; } if (!timestampCombiner.equals(that.timestampCombiner)) { return false; } return namespace.equals(that.namespace); }
@Override public int hashCode() { int result = stateId.hashCode(); result = 31 * result + timestampCombiner.hashCode(); result = 31 * result + namespace.hashCode(); return result; } }
@Override public void add(Instant outputTime) { combinedHold = combinedHold == null ? outputTime : timestampCombiner.combine(combinedHold, outputTime); }
timestampCombiner.assign( currentWindow, windowingStrategy combineFn.addInput(accumulator, nextValue.getValue(), ctxtForInput(nextValue)); windowTimestamp = timestampCombiner.merge( currentWindow, windowTimestamp, currentWindow = nextWindow; windowTimestamp = timestampCombiner.assign( currentWindow, windowFn.getOutputTime(nextValue.getTimestamp(), currentWindow));
@Override public KV<Instant, WindowedValue<KV<K, V>>> map(WindowedValue<KV<K, V>> wv) { BoundedWindow window = Iterables.getOnlyElement(wv.getWindows()); Instant timestamp = timestampCombiner.assign(window, windowFn.getOutputTime(wv.getTimestamp(), window)); return KV.of(timestamp, wv); } }
&& result.getTimestampCombiner().dependsOnlyOnEarliestTimestamp()) { } else if (result.getTimestampCombiner().dependsOnlyOnWindow()) { } else { mergedHold = result.getTimestampCombiner().merge(context.window(), mergedHold, sourceOutputTime);
/** * Shorthand for {@link #merge} with just one element, to place it into the context of a window. * * <p>For example, the {@link #END_OF_WINDOW} policy moves the timestamp to the end of the window. */ public final Instant assign(BoundedWindow intoWindow, Instant timestamp) { return merge(intoWindow, Collections.singleton(timestamp)); }
DisplayData.item("timestampCombiner", getTimestampCombiner().toString()) .withLabel("Timestamp Combiner"));
/** Varargs variant of {@link #combine}. */ public final Instant combine(Instant... timestamps) { return combine(Arrays.asList(timestamps)); }
/** * Return {@code timestamp}, possibly shifted forward in time according to the window strategy's * output time function. */ private Instant shift(Instant timestamp, W window) { Instant shifted = windowingStrategy .getTimestampCombiner() .assign(window, windowingStrategy.getWindowFn().getOutputTime(timestamp, window)); // Don't call checkState(), to avoid calling BoundedWindow.formatTimestamp() every time if (shifted.isBefore(timestamp)) { throw new IllegalStateException( String.format( "TimestampCombiner moved element from %s to earlier time %s for window %s", BoundedWindow.formatTimestamp(timestamp), BoundedWindow.formatTimestamp(shifted), window)); } checkState( timestamp.isAfter(window.maxTimestamp()) || !shifted.isAfter(window.maxTimestamp()), "TimestampCombiner moved element from %s to %s which is beyond end of " + "window %s", timestamp, shifted, window); return shifted; }
/** Varargs variant of {@link #merge}. */ public final Instant merge(BoundedWindow intoWindow, Instant... timestamps) { return merge(intoWindow, Arrays.asList(timestamps)); }
@Test public void testDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData displayData = DisplayData.from(window); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( displayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(displayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(displayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat(displayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
options, sideInputReader, singletonW); Instant windowTimestamp = timestampCombiner.assign( mergedWindow, windowFn.getOutputTime(currentValue.getTimestamp(), mergedWindow)); accumAndInstant = new Tuple2<>(accumT, windowTimestamp); currentValue.getValue().getValue(), options, sideInputReader, singletonW); accumAndInstant.f1 = timestampCombiner.combine( accumAndInstant.f1, timestampCombiner.assign( mergedWindow, windowingStrategy
@Override public void add(Instant value) { try { org.apache.flink.api.common.state.ValueState<Instant> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); Instant current = state.value(); if (current == null) { state.update(value); watermarkHolds.put(namespace.stringKey(), value); } else { Instant combined = timestampCombiner.combine(current, value); state.update(combined); watermarkHolds.put(namespace.stringKey(), combined); } } catch (Exception e) { throw new RuntimeException("Error updating state.", e); } }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testKeyedByTimestamp() { WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10)); BoundedWindow window = new IntervalWindow(new org.joda.time.Instant(0), new org.joda.time.Instant(10)); GroupByKeyTranslator.KeyedByTimestamp keyedByTimestamp = new GroupByKeyTranslator.KeyedByTimestamp(slidingWindows, timestampCombiner); WindowedValue<KV<String, String>> value = WindowedValue.of( KV.of("key", "val"), org.joda.time.Instant.now(), window, PaneInfo.NO_FIRING); KV<org.joda.time.Instant, WindowedValue<KV<String, String>>> result = keyedByTimestamp.map(value); org.joda.time.Instant time = timestampCombiner.assign( window, slidingWindows.getOutputTime(value.getTimestamp(), window)); assertThat(result, equalTo(KV.of(time, value))); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } FlinkWatermarkHoldState<?, ?> that = (FlinkWatermarkHoldState<?, ?>) o; if (!stateId.equals(that.stateId)) { return false; } if (!timestampCombiner.equals(that.timestampCombiner)) { return false; } return namespace.equals(that.namespace); }
Instant mergedTimestamp = timestampCombiner.merge(currentWindow, windowTimestamps); Instant mergedTimestamp = timestampCombiner.merge(currentWindow, windowTimestamps); Iterable<AccumT> accumsToMerge = Iterables.unmodifiableIterable(currentWindowAccumulators); WindowedValue<Iterable<AccumT>> preMergeWindowedValue =
@Override public int hashCode() { int result = address.hashCode(); result = 31 * result + timestampCombiner.hashCode(); result = 31 * result + namespace.hashCode(); return result; } }
@Test @Category(ValidatesRunner.class) public void testPrimitiveDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData primitiveDisplayData = Iterables.getOnlyElement( DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window)); assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( primitiveDisplayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(primitiveDisplayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat( primitiveDisplayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }