public static <InputT, OutputT, RestrictionT> ProcessFnRunner<InputT, OutputT, RestrictionT> newProcessFnRunner( ProcessFn<InputT, OutputT, RestrictionT, ?> fn, PipelineOptions options, Collection<PCollectionView<?>> views, ReadyCheckingSideInputReader sideInputReader, OutputManager outputManager, TupleTag<OutputT> mainOutputTag, List<TupleTag<?>> additionalOutputTags, StepContext stepContext, @Nullable Coder<KeyedWorkItem<byte[], KV<InputT, RestrictionT>>> inputCoder, Map<TupleTag<?>, Coder<?>> outputCoders, WindowingStrategy<?, ?> windowingStrategy) { return new ProcessFnRunner<>( simpleRunner( options, fn, sideInputReader, outputManager, mainOutputTag, additionalOutputTags, stepContext, inputCoder, outputCoders, windowingStrategy), views, sideInputReader); } }
@Override protected DoFnRunner<KeyedWorkItem<K, InputT>, KV<K, OutputT>> createWrappingDoFnRunner( DoFnRunner<KeyedWorkItem<K, InputT>, KV<K, OutputT>> wrappedRunner) { // When the doFn is this, we know it came from WindowDoFnOperator and // InputT = KeyedWorkItem<K, V> // OutputT = KV<K, V> // // for some K, V return DoFnRunners.lateDataDroppingRunner( (DoFnRunner) doFnRunner, timerInternals, windowingStrategy); }
doFnRunner = DoFnRunners.simpleRunner( options, doFn, doFnRunner = DoFnRunners.lateDataDroppingRunner( (DoFnRunner) doFnRunner, stepContext, doFn, stepContext.stateInternals(), windowCoder); doFnRunner = DoFnRunners.defaultStatefulDoFnRunner( doFn, doFnRunner,
DoFnRunners.simpleRunner( pipelineOptions.get(), doFn, DoFnRunners.defaultStatefulDoFnRunner( doFn, doFnRunner, windowingStrategy, cleanupTimer, stateCleaner);
protected DoFnRunner<InputT, OutputT> createWrappingDoFnRunner( DoFnRunner<InputT, OutputT> wrappedRunner) { if (keyCoder != null) { StatefulDoFnRunner.CleanupTimer cleanupTimer = new StatefulDoFnRunner.TimeInternalsCleanupTimer(timerInternals, windowingStrategy); // we don't know the window type @SuppressWarnings({"unchecked", "rawtypes"}) Coder windowCoder = windowingStrategy.getWindowFn().windowCoder(); @SuppressWarnings({"unchecked", "rawtypes"}) StatefulDoFnRunner.StateCleaner<?> stateCleaner = new StatefulDoFnRunner.StateInternalsStateCleaner<>( doFn, keyedStateInternals, windowCoder); return DoFnRunners.defaultStatefulDoFnRunner( doFn, wrappedRunner, windowingStrategy, cleanupTimer, stateCleaner); } else { return doFnRunner; } }
protected DoFnRunner<InputT, OutputT> createWrappingDoFnRunner( DoFnRunner<InputT, OutputT> wrappedRunner) { if (keyCoder != null) { StatefulDoFnRunner.CleanupTimer cleanupTimer = new StatefulDoFnRunner.TimeInternalsCleanupTimer(timerInternals, windowingStrategy); // we don't know the window type @SuppressWarnings({"unchecked", "rawtypes"}) Coder windowCoder = windowingStrategy.getWindowFn().windowCoder(); @SuppressWarnings({"unchecked", "rawtypes"}) StatefulDoFnRunner.StateCleaner<?> stateCleaner = new StatefulDoFnRunner.StateInternalsStateCleaner<>( doFn, keyedStateInternals, windowCoder); return DoFnRunners.defaultStatefulDoFnRunner( doFn, wrappedRunner, windowingStrategy, cleanupTimer, stateCleaner); } else { return doFnRunner; } }
public PushbackSideInputDoFnRunner<InputT, OutputT> createRunner( ReadyCheckingSideInputReader sideInputReader) { PipelineOptions options = serializedOptions.get(); DoFnRunner<InputT, OutputT> underlying = DoFnRunners.simpleRunner( options, fn, sideInputReader, outputManager, mainOutputTag, sideOutputTags, stepContext, null, outputCoders, windowingStrategy); return SimplePushbackSideInputDoFnRunner.create(underlying, sideInputs, sideInputReader); } }
@Test public void testLateDropping() throws Exception { MetricsContainerImpl container = new MetricsContainerImpl("any"); MetricsEnvironment.setCurrentContainer(container); timerInternals.advanceInputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); timerInternals.advanceOutputWatermark(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); DoFn<KV<String, Integer>, Integer> fn = new MyDoFn(); DoFnRunner<KV<String, Integer>, Integer> runner = DoFnRunners.defaultStatefulDoFnRunner( fn, getDoFnRunner(fn), WINDOWING_STRATEGY, new StatefulDoFnRunner.TimeInternalsCleanupTimer(timerInternals, WINDOWING_STRATEGY), new StatefulDoFnRunner.StateInternalsStateCleaner<>( fn, stateInternals, (Coder) WINDOWING_STRATEGY.getWindowFn().windowCoder())); runner.startBundle(); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE)); Instant timestamp = new Instant(0); runner.processElement( WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING)); long droppedValues = container .getCounter( MetricName.named( StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)) .getCumulative(); assertEquals(1L, droppedValues); runner.finishBundle(); }
@Override protected DoFnRunner<KeyedWorkItem<K, InputT>, KV<K, OutputT>> createWrappingDoFnRunner( DoFnRunner<KeyedWorkItem<K, InputT>, KV<K, OutputT>> wrappedRunner) { // When the doFn is this, we know it came from WindowDoFnOperator and // InputT = KeyedWorkItem<K, V> // OutputT = KV<K, V> // // for some K, V return DoFnRunners.lateDataDroppingRunner( (DoFnRunner) doFnRunner, timerInternals, windowingStrategy); }
doFnRunner = DoFnRunners.simpleRunner( options, wrappedDoFn,
DoFnRunners.defaultStatefulDoFnRunner( fn, getDoFnRunner(fn),
DoFnRunners.simpleRunner( options.get(), doFn,
DoFnRunners.simpleRunner( serializedOptions.get(), doFn,
DoFnRunner<InputT, OutputT> doFnRunner = DoFnRunners.simpleRunner( serializedOptions.get(), doFn, new FlinkSideInputReader(sideInputs, runtimeContext),
DoFnRunners.simpleRunner( serializedOptions.get(), doFn,
DoFnRunners.simpleRunner( serializedOptions.get(), dofn,
DoFnRunner<KV<K, V>, OutputT> doFnRunner = DoFnRunners.simpleRunner( serializedOptions.get(), dofn, new FlinkSideInputReader(sideInputs, runtimeContext),
DoFnRunners.simpleRunner( serializedOptions.get(), dofn,
DoFnRunners.simpleRunner( options, doFn,
DoFnRunners.simpleRunner( options, doFn,