/** * Finish bundle without checking for conditions. */ final void forceFinishBundle() { if (!bundleFinished) { bundleFinished = true; if (pushBackRunner == null) { doFnRunner.finishBundle(); } else { pushBackRunner.finishBundle(); } } }
/** * Check whether invoke startBundle, if it is, need to output elements that were buffered as part * of finishing a bundle in snapshot() first. * * <p>In order to avoid having {@link DoFnRunner#processElement(WindowedValue)} or {@link * DoFnRunner#onTimer(String, BoundedWindow, Instant, TimeDomain)} not between StartBundle and * FinishBundle, this method needs to be called in each processElement and each processWatermark * and onProcessingTime. Do not need to call in onEventTime, because it has been guaranteed in the * processWatermark. */ private void checkInvokeStartBundle() { if (!bundleStarted) { outputManager.flushBuffer(); pushbackDoFnRunner.startBundle(); bundleStarted = true; } }
outputManager.clear(); doFnRunner.startBundle(); Iterable<WindowedValue<InputT>> values = doFnRunner.processElementInReadyWindows(value); Iterables.addAll(nextPushedBackValues, values); Iterables.addAll(pushedBackValues, nextPushedBackValues); doFnRunner.finishBundle();
@Override public void fireTimer(Object key, Collection<TimerData> timerDataSet) { pushbackDoFnRunner.startBundle(); @SuppressWarnings("unchecked") Coder<Object> keyCoder = (Coder) currentKeyStateInternals.getKeyCoder(); ((StateInternalsProxy) currentKeyStateInternals).setKey(key); currentKeyTimerInternals.setContext( key, keyCoder, new Instant(this.currentInputWatermark), new Instant(this.currentOutputWatermark)); for (TimerData timerData : timerDataSet) { StateNamespace namespace = timerData.getNamespace(); checkArgument(namespace instanceof WindowNamespace); BoundedWindow window = ((WindowNamespace<?>) namespace).getWindow(); pushbackDoFnRunner.onTimer( timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain()); } pushbackDoFnRunner.finishBundle(); }
/** * handle pushbacks. */ private void handlePushBacks() { // Force-finish, before (possibly) processing pushed-back data. // // Main reason: // {@link org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner} // caches for each bundle the side inputs that are not ready. // We need to re-start the bundle to advertise the (possibly) newly available side input. forceFinishBundle(); // forced // With the new side input added, we may be able to process some pushed-back elements. final List<WindowedValue<InputT>> pushedBackAgain = new ArrayList<>(); long pushedBackAgainWatermark = Long.MAX_VALUE; for (final WindowedValue<InputT> curPushedBack : curPushedBacks) { checkAndInvokeBundle(); final Iterable<WindowedValue<InputT>> pushedBack = getPushBackRunner().processElementInReadyWindows(curPushedBack); checkAndFinishBundle(); for (final WindowedValue<InputT> wv : pushedBack) { pushedBackAgainWatermark = Math.min(pushedBackAgainWatermark, wv.getTimestamp().getMillis()); pushedBackAgain.add(wv); } } curPushedBacks = pushedBackAgain; curPushedBackWatermark = pushedBackAgainWatermark; }
public void fireTimer(InternalTimer<?, TimerData> timer) { TimerInternals.TimerData timerData = timer.getNamespace(); StateNamespace namespace = timerData.getNamespace(); // This is a user timer, so namespace must be WindowNamespace checkArgument(namespace instanceof WindowNamespace); BoundedWindow window = ((WindowNamespace) namespace).getWindow(); pushbackDoFnRunner.onTimer(timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain()); }
private Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) { try { pushbackDoFnRunner.startBundle(); if (currentKeyStateInternals != null) { InputT value = elem.getValue(); pushbackDoFnRunner.processElementInReadyWindows(elem); pushbackDoFnRunner.finishBundle(); return pushedBack; } catch (UserCodeException ue) {
@Override public final void processElement1(StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); Iterable<WindowedValue<InputT>> justPushedBack = pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue()); long min = pushedBackWatermark; for (WindowedValue<InputT> pushedBackValue : justPushedBack) { min = Math.min(min, pushedBackValue.getTimestamp().getMillis()); pushedBackElementsHandler.pushBack(pushedBackValue); } setPushedBackWatermark(min); checkInvokeFinishBundleByCount(); }
public void fireTimer(InternalTimer<?, TimerData> timer) { TimerInternals.TimerData timerData = timer.getNamespace(); StateNamespace namespace = timerData.getNamespace(); // This is a user timer, so namespace must be WindowNamespace checkArgument(namespace instanceof WindowNamespace); BoundedWindow window = ((WindowNamespace) namespace).getWindow(); timerInternals.cleanupPendingTimer(timerData); pushbackDoFnRunner.onTimer( timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain()); }
/** * Checks whether it is time to finish the bundle and finish it. */ final void checkAndFinishBundle() { if (!bundleFinished) { if (currBundleCount >= bundleSize || System.currentTimeMillis() - prevBundleStartTime >= bundleMillis) { bundleFinished = true; if (pushBackRunner == null) { doFnRunner.finishBundle(); } else { pushBackRunner.finishBundle(); } } } }
@Override public final void processElement1(StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); Iterable<WindowedValue<InputT>> justPushedBack = pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue()); long min = pushedBackWatermark; for (WindowedValue<InputT> pushedBackValue : justPushedBack) { min = Math.min(min, pushedBackValue.getTimestamp().getMillis()); pushedBackElementsHandler.pushBack(pushedBackValue); } setPushedBackWatermark(min); checkInvokeFinishBundleByCount(); }
/** * Check whether invoke startBundle, if it is, need to output elements that were buffered as part * of finishing a bundle in snapshot() first. * * <p>In order to avoid having {@link DoFnRunner#processElement(WindowedValue)} or {@link * DoFnRunner#onTimer(String, BoundedWindow, Instant, TimeDomain)} not between StartBundle and * FinishBundle, this method needs to be called in each processElement and each processWatermark * and onProcessingTime. Do not need to call in onEventTime, because it has been guaranteed in the * processWatermark. */ private void checkInvokeStartBundle() { if (!bundleStarted) { outputManager.flushBuffer(); pushbackDoFnRunner.startBundle(); bundleStarted = true; } }
public void fireTimer(InternalTimer<?, TimerData> timer) { TimerInternals.TimerData timerData = timer.getNamespace(); StateNamespace namespace = timerData.getNamespace(); // This is a user timer, so namespace must be WindowNamespace checkArgument(namespace instanceof WindowNamespace); BoundedWindow window = ((WindowNamespace) namespace).getWindow(); timerInternals.cleanupPendingTimer(timer.getNamespace()); pushbackDoFnRunner.onTimer( timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain()); }
protected void invokeFinishBundle() { if (bundleStarted) { pushbackDoFnRunner.finishBundle(); bundleStarted = false; elementCount = 0L; lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime(); } }
pushbackDoFnRunner.processElementInReadyWindows(element); Iterables.addAll(newPushedBack, justPushedBack);
/** * Check whether invoke startBundle, if it is, need to output elements that were * buffered as part of finishing a bundle in snapshot() first. * * <p>In order to avoid having {@link DoFnRunner#processElement(WindowedValue)} or * {@link DoFnRunner#onTimer(String, BoundedWindow, Instant, TimeDomain)} not between * StartBundle and FinishBundle, this method needs to be called in each processElement * and each processWatermark and onProcessingTime. Do not need to call in onEventTime, * because it has been guaranteed in the processWatermark. */ private void checkInvokeStartBundle() { if (!bundleStarted) { outputManager.flushBuffer(); pushbackDoFnRunner.startBundle(); bundleStarted = true; } }
/** Tests that a call to onTimer gets delegated. */ @Test public void testOnTimerCalled() { PushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of()); String timerId = "fooTimer"; IntervalWindow window = new IntervalWindow(new Instant(4), new Instant(16)); Instant timestamp = new Instant(72); // Mocking is not easily compatible with annotation analysis, so we manually record // the method call. runner.onTimer(timerId, window, new Instant(timestamp), TimeDomain.EVENT_TIME); assertThat( underlying.firedTimers, contains( TimerData.of( timerId, StateNamespaces.window(IntervalWindow.getCoder(), window), timestamp, TimeDomain.EVENT_TIME))); }
private void invokeFinishBundle() { if (bundleStarted) { pushbackDoFnRunner.finishBundle(); bundleStarted = false; elementCount = 0L; lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime(); } }
pushbackDoFnRunner.processElementInReadyWindows(element); Iterables.addAll(newPushedBack, justPushedBack);
/** * Checks whether the bundle is finished or not. * Starts the bundle if it is done. * * TODO #263: Partial Combining for Beam Streaming * We may want to use separate methods for doFnRunner/pushBackRunner * (same applies to the other bundle-related methods) */ final void checkAndInvokeBundle() { if (bundleFinished) { bundleFinished = false; if (pushBackRunner == null) { doFnRunner.startBundle(); } else { pushBackRunner.startBundle(); } prevBundleStartTime = System.currentTimeMillis(); currBundleCount = 0; } currBundleCount += 1; }