@Override public void recordContent(StateAccessor<K> state) { state.access(PANE_ADDITIONS_TAG).add(1L); }
@Override public void onElement(OnElementContext c) throws Exception { c.state().access(ELEMENTS_IN_PANE_TAG).add(1L); }
@ProcessElement public void processElement( @TimerId(END_OF_WINDOW_ID) Timer timer, @StateId(BATCH_ID) BagState<InputT> batch, @StateId(NUM_ELEMENTS_IN_BATCH_ID) CombiningState<Long, long[], Long> numElementsInBatch, @StateId(KEY_ID) ValueState<K> key, @Element KV<K, InputT> element, BoundedWindow window, OutputReceiver<KV<K, Iterable<InputT>>> receiver) { Instant windowExpires = window.maxTimestamp().plus(allowedLateness); LOG.debug( "*** SET TIMER *** to point in time {} for window {}", windowExpires.toString(), window.toString()); timer.set(windowExpires); key.write(element.getKey()); batch.add(element.getValue()); LOG.debug("*** BATCH *** Add element for window {} ", window.toString()); // blind add is supported with combiningState numElementsInBatch.add(1L); Long num = numElementsInBatch.read(); if (num % prefetchFrequency == 0) { //prefetch data and modify batch state (readLater() modifies this) batch.readLater(); } if (num >= batchSize) { LOG.debug("*** END OF BATCH *** for window {}", window.toString()); flushBatch(receiver, key, batch, numElementsInBatch); } }
@Test public void testMergeCombiningValueIntoNewNamespace() throws Exception { CombiningState<Integer, int[], Integer> value1 = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR); CombiningState<Integer, int[], Integer> value2 = underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR); CombiningState<Integer, int[], Integer> value3 = underTest.state(NAMESPACE_3, SUM_INTEGER_ADDR); value1.add(5); value2.add(10); value1.add(6); StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value3); // Merging clears the old values and updates the result value. assertThat(value1.read(), equalTo(0)); assertThat(value2.read(), equalTo(0)); assertThat(value3.read(), equalTo(21)); }
/** * Add the given value to the internal side-input store of the given side input. This might change * the result of {@link #isReady(PCollectionView, BoundedWindow)} for that side input. */ public void addSideInputValue(PCollectionView<?> sideInput, WindowedValue<Iterable<?>> value) { @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder(); StateTag<ValueState<Iterable<?>>> stateTag = sideInputContentsTags.get(sideInput); for (BoundedWindow window : value.getWindows()) { stateInternals .state(StateNamespaces.window(windowCoder, window), stateTag) .write(value.getValue()); stateInternals .state(StateNamespaces.global(), availableWindowsTags.get(sideInput)) .add(window); } }
@Test public void testMergeCombiningValueIntoSource() throws Exception { CombiningState<Integer, int[], Integer> value1 = underTest.state(NAMESPACE_1, SUM_INTEGER_ADDR); CombiningState<Integer, int[], Integer> value2 = underTest.state(NAMESPACE_2, SUM_INTEGER_ADDR); value1.add(5); value2.add(10); value1.add(6); assertThat(value1.read(), equalTo(11)); assertThat(value2.read(), equalTo(10)); // Merging clears the old values and updates the result value. StateMerging.mergeCombiningValues(Arrays.asList(value1, value2), value1); assertThat(value1.read(), equalTo(21)); assertThat(value2.read(), equalTo(0)); }
@ProcessElement public void processElement( ProcessContext c, @Element KV<String, Double> element, @StateId(stateId) CombiningState<Double, CountSum<Double>, Double> state, OutputReceiver<String> r) { state.add(element.getValue()); Double currentValue = state.read(); if (Math.abs(currentValue - 0.5) < EPSILON) { r.output("right on"); } } };
@ProcessElement public void processElement( @Element KV<String, Integer> element, @StateId(stateId) CombiningState<Integer, MyInteger, Integer> state, OutputReceiver<String> r) { state.add(element.getValue()); Integer currentValue = state.read(); if (currentValue == EXPECTED_SUM) { r.output("right on"); } } };
@ProcessElement public void processElement( @Element KV<String, Integer> element, @StateId(stateId) CombiningState<Integer, MyInteger, Integer> state, OutputReceiver<String> r) { state.add(element.getValue()); Integer currentValue = state.read(); if (currentValue == EXPECTED_SUM) { r.output("right on"); } } };
@ProcessElement public void processElement( @Element KV<String, KV<String, Integer>> element, @StateId(stateId) MapState<String, MyInteger> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<KV<String, MyInteger>> r) { KV<String, Integer> value = element.getValue(); state.put(value.getKey(), new MyInteger(value.getValue())); count.add(1); if (count.read() >= 4) { Iterable<Map.Entry<String, MyInteger>> iterate = state.entries().read(); for (Map.Entry<String, MyInteger> entry : iterate) { r.output(KV.of(entry.getKey(), entry.getValue())); } } } };
@ProcessElement public void processElement( ProcessContext c, @Element KV<String, KV<String, Integer>> element, @StateId(stateId) MapState<String, MyInteger> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<KV<String, MyInteger>> r) { KV<String, Integer> value = element.getValue(); state.put(value.getKey(), new MyInteger(value.getValue())); count.add(1); if (count.read() >= 4) { Iterable<Map.Entry<String, MyInteger>> iterate = state.entries().read(); for (Map.Entry<String, MyInteger> entry : iterate) { r.output(KV.of(entry.getKey(), entry.getValue())); } } } };
@ProcessElement public void processElement( @Element KV<String, Integer> element, @StateId(stateId) SetState<MyInteger> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<Set<MyInteger>> r) { state.add(new MyInteger(element.getValue())); count.add(1); if (count.read() >= 4) { Set<MyInteger> set = Sets.newHashSet(state.read()); r.output(set); } } };
@ProcessElement public void processElement( @Element KV<String, Integer> element, @StateId(stateId) SetState<MyInteger> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<Set<MyInteger>> r) { state.add(new MyInteger(element.getValue())); count.add(1); if (count.read() >= 4) { Set<MyInteger> set = Sets.newHashSet(state.read()); r.output(set); } } };
@ProcessElement public void processElement( ProcessContext c, @Element KV<String, KV<String, Integer>> element, @StateId(stateId) MapState<String, Integer> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<KV<String, Integer>> r) { KV<String, Integer> value = element.getValue(); ReadableState<Iterable<Entry<String, Integer>>> entriesView = state.entries(); state.put(value.getKey(), value.getValue()); count.add(1); if (count.read() >= 4) { Iterable<Map.Entry<String, Integer>> iterate = state.entries().read(); // Make sure that the cached Iterable doesn't change when new elements are added, // but that cached ReadableState views of the state do change. state.put("BadKey", -1); assertEquals(3, Iterables.size(iterate)); assertEquals(4, Iterables.size(entriesView.read())); assertEquals(4, Iterables.size(state.entries().read())); for (Map.Entry<String, Integer> entry : iterate) { r.output(KV.of(entry.getKey(), entry.getValue())); } } } };
@ProcessElement public void processElement( @Element KV<String, Integer> element, @StateId(stateId) SetState<Integer> state, @StateId(countStateId) CombiningState<Integer, int[], Integer> count, OutputReceiver<Set<Integer>> r) { ReadableState<Boolean> isEmpty = state.isEmpty(); state.add(element.getValue()); assertFalse(isEmpty.read()); count.add(1); if (count.read() >= 4) { // Make sure that the cached Iterable doesn't change when new elements are added. Iterable<Integer> ints = state.read(); state.add(-1); assertEquals(3, Iterables.size(ints)); assertEquals(4, Iterables.size(state.read())); Set<Integer> set = Sets.newHashSet(ints); r.output(set); } } };