InMemorySingleKeyBagState(String userStateId, Coder<V> valueCoder, Coder<W> windowCoder) { this.windowCoder = windowCoder; this.stateTag = StateTags.bag(userStateId, valueCoder); }
InMemorySingleKeyBagState(String userStateId, Coder<V> valueCoder, Coder<W> windowCoder) { this.windowCoder = windowCoder; this.stateTag = StateTags.bag(userStateId, valueCoder); }
/** * Create a factory that produces {@link SystemReduceFn} instances that that buffer all of the * input values in persistent state and produces an {@code Iterable<T>}. */ public static <K, T, W extends BoundedWindow> SystemReduceFn<K, T, Iterable<T>, Iterable<T>, W> buffering(final Coder<T> inputCoder) { final StateTag<BagState<T>> bufferTag = StateTags.makeSystemTagInternal(StateTags.bag(BUFFER_NAME, inputCoder)); return new SystemReduceFn<K, T, Iterable<T>, Iterable<T>, W>(bufferTag) { @Override public void prefetchOnMerge(MergingStateAccessor<K, W> state) throws Exception { StateMerging.prefetchBags(state, bufferTag); } @Override public void onMerge(OnMergeContext c) throws Exception { StateMerging.mergeBags(c.state(), bufferTag); } }; }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
BufferedOutputManager( Output<StreamRecord<WindowedValue<OutputT>>> output, TupleTag<OutputT> mainTag, Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags, final Map<TupleTag<?>, Coder<WindowedValue<?>>> tagsToCoders, Map<TupleTag<?>, Integer> tagsToIds, StateInternals stateInternals) { this.output = output; this.mainTag = mainTag; this.tagsToOutputTags = tagsToOutputTags; this.tagsToIds = tagsToIds; this.idsToTags = new HashMap<>(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToTags.put(entry.getValue(), entry.getKey()); } ImmutableMap.Builder<Integer, Coder<WindowedValue<?>>> idsToCodersBuilder = ImmutableMap.builder(); for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) { idsToCodersBuilder.put(entry.getValue(), tagsToCoders.get(entry.getKey())); } StateTag<BagState<KV<Integer, WindowedValue<?>>>> bufferTag = StateTags.bag("bundle-buffer-tag", new TaggedKvCoder(idsToCodersBuilder.build())); bufferState = stateInternals.state(StateNamespaces.global(), bufferTag); }
@Test public void testBagEquality() { StateTag<?> fooVarInt1 = StateTags.bag("foo", VarIntCoder.of()); StateTag<?> fooVarInt2 = StateTags.bag("foo", VarIntCoder.of()); StateTag<?> fooBigEndian = StateTags.bag("foo", BigEndianIntegerCoder.of()); StateTag<?> barVarInt = StateTags.bag("bar", VarIntCoder.of()); assertEquals(fooVarInt1, fooVarInt2); assertNotEquals(fooVarInt1, fooBigEndian); assertNotEquals(fooVarInt1, barVarInt); }
pushedBackTag = StateTags.bag("pushed-back-values", inputCoder);