/** * handle pushbacks. */ private void handlePushBacks() { // Force-finish, before (possibly) processing pushed-back data. // // Main reason: // {@link org.apache.beam.runners.core.SimplePushbackSideInputDoFnRunner} // caches for each bundle the side inputs that are not ready. // We need to re-start the bundle to advertise the (possibly) newly available side input. forceFinishBundle(); // forced // With the new side input added, we may be able to process some pushed-back elements. final List<WindowedValue<InputT>> pushedBackAgain = new ArrayList<>(); long pushedBackAgainWatermark = Long.MAX_VALUE; for (final WindowedValue<InputT> curPushedBack : curPushedBacks) { checkAndInvokeBundle(); final Iterable<WindowedValue<InputT>> pushedBack = getPushBackRunner().processElementInReadyWindows(curPushedBack); checkAndFinishBundle(); for (final WindowedValue<InputT> wv : pushedBack) { pushedBackAgainWatermark = Math.min(pushedBackAgainWatermark, wv.getTimestamp().getMillis()); pushedBackAgain.add(wv); } } curPushedBacks = pushedBackAgain; curPushedBackWatermark = pushedBackAgainWatermark; }
@Override public final void processElement1(StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); Iterable<WindowedValue<InputT>> justPushedBack = pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue()); long min = pushedBackWatermark; for (WindowedValue<InputT> pushedBackValue : justPushedBack) { min = Math.min(min, pushedBackValue.getTimestamp().getMillis()); pushedBackElementsHandler.pushBack(pushedBackValue); } setPushedBackWatermark(min); checkInvokeFinishBundleByCount(); }
@Override public final void processElement1(StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); Iterable<WindowedValue<InputT>> justPushedBack = pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue()); long min = pushedBackWatermark; for (WindowedValue<InputT> pushedBackValue : justPushedBack) { min = Math.min(min, pushedBackValue.getTimestamp().getMillis()); pushedBackElementsHandler.pushBack(pushedBackValue); } setPushedBackWatermark(min); checkInvokeFinishBundleByCount(); }
pushbackDoFnRunner.processElementInReadyWindows(element); Iterables.addAll(newPushedBack, justPushedBack);
pushbackDoFnRunner.processElementInReadyWindows(element); Iterables.addAll(newPushedBack, justPushedBack);
@Override public void onData(final WindowedValue data) { // Need to distinguish side/main inputs and push-back main inputs. if (data.getValue() instanceof SideInputElement) { // This element is a Side Input // TODO #287: Consider Explicit Multi-Input IR Transform final WindowedValue<SideInputElement> sideInputElement = (WindowedValue<SideInputElement>) data; final PCollectionView view = getSideInputs().get(sideInputElement.getValue().getSideInputIndex()); getSideInputReader().addSideInputElement(view, data); handlePushBacks(); // See if we can emit a new watermark, as we may have processed some pushed-back elements onWatermark(new Watermark(curInputWatermark)); } else { // This element is the Main Input checkAndInvokeBundle(); final Iterable<WindowedValue<InputT>> pushedBack = getPushBackRunner().processElementInReadyWindows(data); for (final WindowedValue wv : pushedBack) { curPushedBackWatermark = Math.min(curPushedBackWatermark, wv.getTimestamp().getMillis()); curPushedBacks.add(wv); } checkAndFinishBundle(); } }
@Override public final void processElement1( StreamRecord<WindowedValue<InputT>> streamRecord) throws Exception { checkInvokeStartBundle(); Iterable<WindowedValue<InputT>> justPushedBack = pushbackDoFnRunner.processElementInReadyWindows(streamRecord.getValue()); BagState<WindowedValue<InputT>> pushedBack = nonKeyedStateInternals.state(StateNamespaces.global(), pushedBackTag); checkInitPushedBackWatermark(); long min = pushedBackWatermark.get(); for (WindowedValue<InputT> pushedBackValue : justPushedBack) { min = Math.min(min, pushedBackValue.getTimestamp().getMillis()); pushedBack.add(pushedBackValue); } setPushedBackWatermark(min); checkInvokeFinishBundleByCount(); }
Iterable<WindowedValue<InputT>> values = doFnRunner.processElementInReadyWindows(value); Iterables.addAll(nextPushedBackValues, values);
pushbackDoFnRunner.processElementInReadyWindows(elem); Iterables.addAll(newPushedBack, justPushedBack);
pushbackDoFnRunner.processElementInReadyWindows(elem); pushbackDoFnRunner.finishBundle(); return pushedBack;