/** Return the current data hold, or null if none. Does not clear. For debugging only. */ @Nullable public Instant getDataCurrent(ReduceFn<?, ?, ?, W>.Context context) { return context.state().access(elementHoldTag).read(); } }
/** * Get the earliest watermark hold in this table. Ignores the contents of any underlying table. */ private Instant getEarliestWatermarkHold() { Instant earliest = BoundedWindow.TIMESTAMP_MAX_VALUE; for (State existingState : this.values()) { if (existingState instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) existingState).read(); if (hold != null && hold.isBefore(earliest)) { earliest = hold; } } } return earliest; }
/** Return the earliest output watermark hold in state, or null if none. */ public Instant earliestWatermarkHold() { Instant minimum = null; for (State storage : inMemoryState.values()) { if (storage instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) storage).read(); if (minimum == null || (hold != null && hold.isBefore(minimum))) { minimum = hold; } } } return minimum; } }
/** * Get the earliest watermark hold in this table. Ignores the contents of any underlying table. */ private Instant getEarliestWatermarkHold() { Instant earliest = BoundedWindow.TIMESTAMP_MAX_VALUE; for (State existingState : this.values()) { if (existingState instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) existingState).read(); if (hold != null && hold.isBefore(earliest)) { earliest = hold; } } } return earliest; }
public Instant readThroughAndGetEarliestHold(StateTable readTo) { Instant earliestHold = BoundedWindow.TIMESTAMP_MAX_VALUE; for (StateNamespace namespace : underlying.getNamespacesInUse()) { for (Map.Entry<StateTag, State> existingState : underlying.getTagsInUse(namespace).entrySet()) { if (!((InMemoryState<?>) existingState.getValue()).isCleared()) { // Only read through non-cleared values to ensure that completed windows are // eventually discarded, and remember the earliest watermark hold from among those // values. State state = readTo.get(namespace, existingState.getKey(), StateContexts.nullContext()); if (state instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) state).read(); if (hold != null && hold.isBefore(earliestHold)) { earliestHold = hold; } } } } } return earliestHold; }
public Instant readThroughAndGetEarliestHold(StateTable readTo) { Instant earliestHold = BoundedWindow.TIMESTAMP_MAX_VALUE; for (StateNamespace namespace : underlying.getNamespacesInUse()) { for (Map.Entry<StateTag, State> existingState : underlying.getTagsInUse(namespace).entrySet()) { if (!((InMemoryState<?>) existingState.getValue()).isCleared()) { // Only read through non-cleared values to ensure that completed windows are // eventually discarded, and remember the earliest watermark hold from among those // values. State state = readTo.get(namespace, existingState.getKey(), StateContexts.nullContext()); if (state instanceof WatermarkHoldState) { Instant hold = ((WatermarkHoldState) state).read(); if (hold != null && hold.isBefore(earliestHold)) { earliestHold = hold; } } } } } return earliestHold; }
@Test public void testWatermarkEndOfWindowState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EOW_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_EOW_ADDR), equalTo(value)); }
@Test public void testWatermarkLatestState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_LATEST_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(3000)); assertThat(value.read(), equalTo(new Instant(3000))); value.add(new Instant(1000)); assertThat(value.read(), equalTo(new Instant(3000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_LATEST_ADDR), equalTo(value)); }
@Test public void testWatermarkEarliestState() throws Exception { WatermarkHoldState value = underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR); // State instances are cached, but depend on the namespace. assertEquals(value, underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR)); assertFalse(value.equals(underTest.state(NAMESPACE_2, WATERMARK_EARLIEST_ADDR))); assertThat(value.read(), Matchers.nullValue()); value.add(new Instant(2000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(3000)); assertThat(value.read(), equalTo(new Instant(2000))); value.add(new Instant(1000)); assertThat(value.read(), equalTo(new Instant(1000))); value.clear(); assertThat(value.read(), equalTo(null)); assertThat(underTest.state(NAMESPACE_1, WATERMARK_EARLIEST_ADDR), equalTo(value)); }
@Test public void testWatermarkHoldStateWithUnderlying() { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); TimestampCombiner timestampCombiner = TimestampCombiner.EARLIEST; StateNamespace namespace = new StateNamespaceForTest("foo"); StateTag<WatermarkHoldState> stateTag = StateTags.watermarkStateInternal("wmstate", timestampCombiner); WatermarkHoldState underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), nullValue()); underlyingValue.add(new Instant(250L)); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); WatermarkHoldState copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(100L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(500L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); WatermarkHoldState reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
@Test public void testWatermarkHoldStateWithUnderlying() { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); TimestampCombiner timestampCombiner = TimestampCombiner.EARLIEST; StateNamespace namespace = new StateNamespaceForTest("foo"); StateTag<WatermarkHoldState> stateTag = StateTags.watermarkStateInternal("wmstate", timestampCombiner); WatermarkHoldState underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), nullValue()); underlyingValue.add(new Instant(250L)); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); WatermarkHoldState copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(100L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); assertThat(underlyingValue.read(), equalTo(new Instant(250L))); copyOnAccessState.add(new Instant(500L)); assertThat(copyOnAccessState.read(), equalTo(new Instant(100L))); WatermarkHoldState reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }