return global(); if (parts.size() > 3) { int index = Integer.parseInt(parts.get(2), WindowAndTriggerNamespace.TRIGGER_RADIX); return windowAndTrigger(windowCoder, window, index); } else { return window(windowCoder, window);
private void assertStringKeyRoundTrips( Coder<? extends BoundedWindow> coder, StateNamespace namespace) { assertEquals(namespace, StateNamespaces.fromString(namespace.stringKey(), coder)); } }
/** * Add the given value to the internal side-input store of the given side input. This might change * the result of {@link #isReady(PCollectionView, BoundedWindow)} for that side input. */ public void addSideInputValue(PCollectionView<?> sideInput, WindowedValue<Iterable<?>> value) { @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder(); StateTag<ValueState<Iterable<?>>> stateTag = sideInputContentsTags.get(sideInput); for (BoundedWindow window : value.getWindows()) { stateInternals .state(StateNamespaces.window(windowCoder, window), stateTag) .write(value.getValue()); stateInternals .state(StateNamespaces.global(), availableWindowsTags.get(sideInput)) .add(window); } }
@Override public boolean isReady(PCollectionView<?> sideInput, BoundedWindow window) { Set<BoundedWindow> readyWindows = stateInternals.state(StateNamespaces.global(), availableWindowsTags.get(sideInput)).read(); boolean result = readyWindows != null && readyWindows.contains(window); return result; }
protected StateNamespace namespaceFor(W window) { return StateNamespaces.window(windowCoder, window); }
@Test public void testFromStringIntervalWindow() { assertStringKeyRoundTrips( intervalCoder, StateNamespaces.window(intervalCoder, intervalWindow(1000, 8000))); assertStringKeyRoundTrips( intervalCoder, StateNamespaces.window(intervalCoder, intervalWindow(1000, 8000))); assertStringKeyRoundTrips( intervalCoder, StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 8000), 18)); assertStringKeyRoundTrips( intervalCoder, StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 8000), 19)); assertStringKeyRoundTrips( intervalCoder, StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(2000, 8000), 19)); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Override public void append(K key, W window, Iterator<V> values) { initStateInternals(key); StateNamespace namespace = StateNamespaces.window(windowCoder, window); BagState<V> bagState = stateInternals.state(namespace, stateTag); while (values.hasNext()) { bagState.add(values.next()); } }
@Test public void testTimerDataCoder() throws Exception { CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(GlobalWindow.Coder.INSTANCE), TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME)); Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder(); CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(windowCoder), TimerData.of( "another-id", StateNamespaces.window( windowCoder, new IntervalWindow(new Instant(0), new Instant(100))), new Instant(99), TimeDomain.PROCESSING_TIME)); }
@Test public void testFromStringGlobalWindow() { assertStringKeyRoundTrips(GlobalWindow.Coder.INSTANCE, StateNamespaces.global()); assertStringKeyRoundTrips( GlobalWindow.Coder.INSTANCE, StateNamespaces.window(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE)); assertStringKeyRoundTrips( GlobalWindow.Coder.INSTANCE, StateNamespaces.windowAndTrigger(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, 18)); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Override public void append(K key, W window, Iterator<V> values) { initStateInternals(key); StateNamespace namespace = StateNamespaces.window(windowCoder, window); BagState<V> bagState = stateInternals.state(namespace, stateTag); while (values.hasNext()) { bagState.add(values.next()); } }
@Override public TimerData decode(InputStream inStream) throws CoderException, IOException { String timerId = STRING_CODER.decode(inStream); StateNamespace namespace = StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder); Instant timestamp = INSTANT_CODER.decode(inStream); TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream)); return TimerData.of(timerId, namespace, timestamp, domain); }
/** Test that WindowAndTrigger namespaces are prefixed by the related Window namespace. */ @Test public void testGlobalWindowPrefixing() { StateNamespace window = StateNamespaces.window(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE); StateNamespace windowAndTrigger = StateNamespaces.windowAndTrigger(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, 57); assertThat(windowAndTrigger.stringKey(), Matchers.startsWith(window.stringKey())); assertThat( StateNamespaces.global().stringKey(), Matchers.not(Matchers.startsWith(window.stringKey()))); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Override public void setForWindow(BoundedWindow window) { Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy); // make sure this fires after any window.maxTimestamp() timers gcTime = gcTime.plus(GC_DELAY_MS); timerInternals.setTimer( StateNamespaces.window(windowCoder, window), GC_TIMER_ID, gcTime, TimeDomain.EVENT_TIME); }
/** Test that WindowAndTrigger namespaces are prefixed by the related Window namespace. */ @Test public void testIntervalWindowPrefixing() { StateNamespace window = StateNamespaces.window(intervalCoder, intervalWindow(1000, 87392)); StateNamespace windowAndTrigger = StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 87392), 57); assertThat(windowAndTrigger.stringKey(), Matchers.startsWith(window.stringKey())); assertThat( StateNamespaces.global().stringKey(), Matchers.not(Matchers.startsWith(window.stringKey()))); }
private void checkInitPushedBackWatermark() { // init and restore from pushedBack state. // Not done in initializeState, because OperatorState is not ready. if (!pushedBackWatermark.isPresent()) { BagState<WindowedValue<InputT>> pushedBack = nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag); long min = Long.MAX_VALUE; for (WindowedValue<InputT> value : pushedBack.read()) { min = Math.min(min, value.getTimestamp().getMillis()); } setPushedBackWatermark(min); } }
@Override public void clearForWindow(W window) { for (Map.Entry<String, DoFnSignature.StateDeclaration> entry : signature.stateDeclarations().entrySet()) { try { StateSpec<?> spec = (StateSpec<?>) entry.getValue().field().get(fn); State state = stateInternals.state( StateNamespaces.window(windowCoder, window), StateTags.tagForSpec(entry.getKey(), (StateSpec) spec)); state.clear(); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } } }
/** * This test should not be changed. It verifies that the stringKey matches certain expectations. * If this changes, the ability to reload any pipeline that has persisted these namespaces will be * impacted. */ @Test public void testStability() { StateNamespace global = StateNamespaces.global(); StateNamespace intervalWindow = StateNamespaces.window(intervalCoder, intervalWindow(1000, 87392)); StateNamespace intervalWindowAndTrigger = StateNamespaces.windowAndTrigger(intervalCoder, intervalWindow(1000, 87392), 57); StateNamespace globalWindow = StateNamespaces.window(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE); StateNamespace globalWindowAndTrigger = StateNamespaces.windowAndTrigger(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE, 12); assertEquals("/", global.stringKey()); assertEquals("/gAAAAAABVWD4ogU/", intervalWindow.stringKey()); assertEquals("/gAAAAAABVWD4ogU/1L/", intervalWindowAndTrigger.stringKey()); assertEquals("//", globalWindow.stringKey()); assertEquals("//C/", globalWindowAndTrigger.stringKey()); }