/** Return the current data hold, or null if none. Does not clear. For debugging only. */ @Nullable public Instant getDataCurrent(ReduceFn<?, ?, ?, W>.Context context) { return context.state().access(elementHoldTag).read(); } }
/** Clear any remaining holds. */ public void clearHolds(ReduceFn<?, ?, ?, W>.Context context) { WindowTracing.debug( "WatermarkHold.clearHolds: For key:{}; window:{}; inputWatermark:{}; outputWatermark:{}", context.key(), context.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); context.state().access(elementHoldTag).clear(); context.state().access(EXTRA_HOLD_TAG).clear(); }
/** Prefetch watermark holds in preparation for merging. */ public void prefetchOnMerge(MergingStateAccessor<?, W> context) { Map<W, WatermarkHoldState> map = context.accessInEachMergingWindow(elementHoldTag); WatermarkHoldState result = context.access(elementHoldTag); if (map.isEmpty()) { // Nothing to prefetch. return; } if (map.size() == 1 && map.values().contains(result) && result.getTimestampCombiner().dependsOnlyOnEarliestTimestamp()) { // Nothing to merge if our source and sink is the same. return; } if (result.getTimestampCombiner().dependsOnlyOnWindow()) { // No need to read existing holds since we will just clear. return; } // Prefetch. for (WatermarkHoldState source : map.values()) { source.readLater(); } }
@Test public void testWatermarkLatestState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(3000)); assertThat(value.read(), equalTo(new Instant(3000))); value.add(new Instant(1000)); assertThat(value.read(), equalTo(new Instant(3000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), equalTo(value)); }
@Test public void testWatermarkHoldStateWithUnderlying() { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); TimestampCombiner timestampCombiner = TimestampCombiner.EARLIEST; StateNamespace namespace = new StateNamespaceForTest("foo"); StateTag<WatermarkHoldState> stateTag = StateTags.watermarkStateInternal("wmstate", timestampCombiner); WatermarkHoldState underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), nullValue()); underlyingValue.add(new Instant(250L)); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); WatermarkHoldState copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(100L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(500L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); WatermarkHoldState reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
@Test public void testGetEarliestHoldBeforeCommit() { CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); internals .state( StateNamespaces.global(), StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST)) .add(new Instant(1234L)); thrown.expect(IllegalStateException.class); thrown.expectMessage(CopyOnAccessInMemoryStateInternals.class.getSimpleName()); thrown.expectMessage("Can't get the earliest watermark hold"); thrown.expectMessage("before it is committed"); internals.getEarliestWatermarkHold(); } }
&& result.getTimestampCombiner().dependsOnlyOnEarliestTimestamp()) { } else if (result.getTimestampCombiner().dependsOnlyOnWindow()) { source.clear(); source.readLater(); } else { mergedHold = result.getTimestampCombiner().merge(context.window(), mergedHold, sourceOutputTime); source.clear(); result.add(mergedHold);
@Test public void testWatermarkStateIsEmpty() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR); assertThat(value.isEmpty().read(), Matchers.is(true)); ReadableState<Boolean> readFuture = value.isEmpty(); value.add(new Instant(1000)); assertThat(readFuture.read(), Matchers.is(false)); value.clear(); assertThat(readFuture.read(), Matchers.is(true)); }
@Test public void testWatermarkEarliestState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(3000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(1000)); assertThat(value.read(), equalTo(new Instant(1000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), equalTo(value)); }
@Test public void testWatermarkHoldStateWithUnderlying() { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); TimestampCombiner timestampCombiner = TimestampCombiner.EARLIEST; StateNamespace namespace = new StateNamespaceForTest("foo"); StateTag<WatermarkHoldState> stateTag = StateTags.watermarkStateInternal("wmstate", timestampCombiner); WatermarkHoldState underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), nullValue()); underlyingValue.add(new Instant(250L)); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); WatermarkHoldState copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(100L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(500L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); WatermarkHoldState reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
"Element hold %s is beyond end-of-time", elementHold); context.state().access(elementHoldTag).add(elementHold);
@Test public void testWatermarkEndOfWindowState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), equalTo(value)); }
/** * Get the earliest watermark hold in this table. Ignores the contents of any underlying table. */ private Instant getEarliestWatermarkHold() { Instant earliest = BoundedWindow.TIMESTAMP_MAX_VALUE; for (State existingState : this.values()) { if (existingState instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) existingState).read(); if (hold != null && hold.isBefore(earliest)) { earliest = hold; } } } return earliest; }
@Test public void testGetEarliestHoldBeforeCommit() { CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); internals .state( StateNamespaces.global(), StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST)) .add(new Instant(1234L)); thrown.expect(IllegalStateException.class); thrown.expectMessage(CopyOnAccessInMemoryStateInternals.class.getSimpleName()); thrown.expectMessage("Can't get the earliest watermark hold"); thrown.expectMessage("before it is committed"); internals.getEarliestWatermarkHold(); } }
/** Return the earliest output watermark hold in state, or null if none. */ public Instant earliestWatermarkHold() { Instant minimum = null; for (State storage : inMemoryState.values()) { if (storage instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) storage).read(); if (minimum == null || (hold != null && hold.isBefore(minimum))) { minimum = hold; } } } return minimum; } }