InMemorySingleKeyBagState(String userStateId, Coder<V> valueCoder, Coder<W> windowCoder) { this.windowCoder = windowCoder; this.stateTag = StateTags.bag(userStateId, valueCoder); }
@Override public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT> bindCombining( String id, StateSpec<CombiningState<InputT, AccumT, OutputT>> spec, Coder<AccumT> accumCoder, CombineFn<InputT, AccumT, OutputT> combineFn) { return binder.bindCombiningValue(tagForSpec(id, spec), accumCoder, combineFn); }
/** Return tag for state containing the output watermark hold used for elements. */ public static <W extends BoundedWindow> StateTag<WatermarkHoldState> watermarkHoldTagForTimestampCombiner( TimestampCombiner timestampCombiner) { return StateTags.makeSystemTagInternal( StateTags.<W>watermarkStateInternal("hold", timestampCombiner)); }
/** * Create a factory that produces {@link SystemReduceFn} instances that that buffer all of the * input values in persistent state and produces an {@code Iterable<T>}. */ public static <K, T, W extends BoundedWindow> SystemReduceFn<K, T, Iterable<T>, Iterable<T>, W> buffering(final Coder<T> inputCoder) { final StateTag<BagState<T>> bufferTag = StateTags.makeSystemTagInternal(StateTags.bag(BUFFER_NAME, inputCoder)); return new SystemReduceFn<K, T, Iterable<T>, Iterable<T>, W>(bufferTag) { @Override public void prefetchOnMerge(MergingStateAccessor<K, W> state) throws Exception { StateMerging.prefetchBags(state, bufferTag); } @Override public void onMerge(OnMergeContext c) throws Exception { StateMerging.mergeBags(c.state(), bufferTag); } }; }
if (combineFn.getFn() instanceof CombineFnWithContext) { bufferTag = StateTags.makeSystemTagInternal( StateTags.combiningValueWithContext( BUFFER_NAME, combineFn.getAccumulatorCoder(), StateTags.makeSystemTagInternal( StateTags.combiningValue( BUFFER_NAME, combineFn.getAccumulatorCoder(),
public MergingActiveWindowSet(WindowFn<Object, W> windowFn, StateInternals state) { this.windowFn = windowFn; StateTag<ValueState<Map<W, Set<W>>>> tag = StateTags.makeSystemTagInternal( StateTags.value( "tree", MapCoder.of(windowFn.windowCoder(), SetCoder.of(windowFn.windowCoder())))); valueState = state.state(StateNamespaces.global(), tag); // Little use trying to prefetch this state since the ReduceFnRunner // is stymied until it is available. activeWindowToStateAddressWindows = emptyIfNull(valueState.read()); originalActiveWindowToStateAddressWindows = deepCopy(activeWindowToStateAddressWindows); }
Combine.BinaryCombineIntegerFn minFn = Min.ofIntegers(); StateTag<?> fooCoder1Max1 = StateTags.combiningValueFromInputInternal("foo", input1, maxFn); StateTag<?> fooCoder1Max2 = StateTags.combiningValueFromInputInternal("foo", input1, maxFn); StateTag<?> fooCoder1Min = StateTags.combiningValueFromInputInternal("foo", input1, minFn); StateTag<?> fooCoder2Max = StateTags.combiningValueFromInputInternal("foo", input2, maxFn); StateTag<?> barCoder1Max = StateTags.combiningValueFromInputInternal("bar", input1, maxFn); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder1Max2)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder1Min)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder2Max)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) barCoder1Max));
StateTags.makeSystemTagInternal( StateTags.combiningValueFromInputInternal( "delayed", InstantCoder.of(), Min.naturalOrder()));
StateTags.combiningValueWithContext("foo", accum1, CombineFnUtil.toFnWithContext(maxFn)); StateTag<?> fooCoder1Max2 = StateTags.combiningValueWithContext("foo", accum1, CombineFnUtil.toFnWithContext(maxFn)); StateTag<?> fooCoder1Min = StateTags.combiningValueWithContext("foo", accum1, CombineFnUtil.toFnWithContext(minFn)); StateTags.combiningValueWithContext("foo", accum2, CombineFnUtil.toFnWithContext(maxFn)); StateTag<?> barCoder1Max = StateTags.combiningValueWithContext("bar", accum1, CombineFnUtil.toFnWithContext(maxFn)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder1Max2)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder1Min)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) fooCoder2Max)); StateTags.convertToBagTagInternal((StateTag) fooCoder1Max1), StateTags.convertToBagTagInternal((StateTag) barCoder1Max));
StateTags.combiningValue( "side-input-available-windows-" + sideInput.getTagInternal().getId(), SetCoder.of(windowCoder), StateTags.value( "side-input-data-" + sideInput.getTagInternal().getId(), (Coder) IterableCoder.of(sideInput.getCoderInternal()));
public ProcessFn( DoFn<InputT, OutputT> fn, Coder<InputT> elementCoder, Coder<RestrictionT> restrictionCoder, WindowingStrategy<InputT, ?> inputWindowingStrategy) { this.fn = fn; this.elementCoder = elementCoder; this.restrictionCoder = restrictionCoder; this.inputWindowingStrategy = inputWindowingStrategy; this.elementTag = StateTags.value( "element", WindowedValue.getFullCoder( elementCoder, inputWindowingStrategy.getWindowFn().windowCoder())); this.restrictionTag = StateTags.value("restriction", restrictionCoder); }
@Test public void testWatermarkBagEquality() { StateTag<?> foo1 = StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST); StateTag<?> foo2 = StateTags.watermarkStateInternal("foo", TimestampCombiner.EARLIEST); StateTag<?> bar = StateTags.watermarkStateInternal("bar", TimestampCombiner.EARLIEST); StateTag<?> bar2 = StateTags.watermarkStateInternal("bar", TimestampCombiner.LATEST); // Same id, same fn. assertEquals(foo1, foo2); // Different id, same fn. assertNotEquals(foo1, bar); // Same id, different fn. assertEquals(bar, bar2); }
@Test public void testValueEquality() { StateTag<?> fooVarInt1 = StateTags.value("foo", VarIntCoder.of()); StateTag<?> fooVarInt2 = StateTags.value("foo", VarIntCoder.of()); StateTag<?> fooBigEndian = StateTags.value("foo", BigEndianIntegerCoder.of()); StateTag<?> barVarInt = StateTags.value("bar", VarIntCoder.of()); assertEquals(fooVarInt1, fooVarInt2); assertNotEquals(fooVarInt1, fooBigEndian); assertNotEquals(fooVarInt1, barVarInt); }
InMemorySingleKeyBagState(String userStateId, Coder<V> valueCoder, Coder<W> windowCoder) { this.windowCoder = windowCoder; this.stateTag = StateTags.bag(userStateId, valueCoder); }
@Override public <KeyT, ValueT> MapState<KeyT, ValueT> bindMap( String id, StateSpec<MapState<KeyT, ValueT>> spec, Coder<KeyT> mapKeyCoder, Coder<ValueT> mapValueCoder) { return binder.bindMap(tagForSpec(id, spec), mapKeyCoder, mapValueCoder); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Override public <InputT, AccumT, OutputT> CombiningState<InputT, AccumT, OutputT> bindCombiningWithContext( String id, StateSpec<CombiningState<InputT, AccumT, OutputT>> spec, Coder<AccumT> accumCoder, CombineFnWithContext<InputT, AccumT, OutputT> combineFn) { return binder.bindCombiningValueWithContext(tagForSpec(id, spec), accumCoder, combineFn); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Override public WatermarkHoldState bindWatermark( String id, StateSpec<WatermarkHoldState> spec, TimestampCombiner timestampCombiner) { return binder.bindWatermark(tagForSpec(id, spec), timestampCombiner); } };
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }