private DoFnRunner<KV<String, Integer>, Integer> getDoFnRunner( DoFn<KV<String, Integer>, Integer> fn) { return new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WINDOWING_STRATEGY); }
public static NullSideInputReader of(Iterable<? extends PCollectionView<?>> views) { return new NullSideInputReader(views); }
ApexStreamTuple.Logging.isDebugEnabled( pipelineOptions.get().as(ApexPipelineOptions.class), this); SideInputReader sideInputReader = NullSideInputReader.of(sideInputs); if (!sideInputs.isEmpty()) { sideInputHandler = new SideInputHandler(sideInputs, sideInputStateInternals);
setCurrentOutputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()); sideInputReader = NullSideInputReader.of(sideInputs);
setCurrentOutputWatermark(BoundedWindow.TIMESTAMP_MIN_VALUE.getMillis()); sideInputReader = NullSideInputReader.of(sideInputs);
@Test public void testStartBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.startBundle(); }
public static NullSideInputReader empty() { return new NullSideInputReader(Collections.emptySet()); }
sideInputReader = NullSideInputReader.of(sideInputs);
@Test public void testFinishBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.finishBundle(); }
@Test public void testProcessElementExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.processElement(WindowedValue.valueInGlobalWindow("anyValue")); }
@Test public void testOnTimerExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.onTimer( ThrowingDoFn.TIMER_ID, GlobalWindow.INSTANCE, new Instant(0), TimeDomain.EVENT_TIME); }
NullSideInputReader.empty(), SystemReduceFn.buffering(this.valueCoder), serializedOptions.get());
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link * TriggerStateMachine}, for mocking the interactions between {@link ReduceFnRunner} and the * {@link TriggerStateMachine}. * * <p>Ignores the {@link Trigger} on the {@link WindowingStrategy}. */ public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowingStrategy<?, W> windowingStrategy, TriggerStateMachine triggerStateMachine) throws Exception { return new ReduceFnTester<>( windowingStrategy, triggerStateMachine, SystemReduceFn.buffering(VarIntCoder.of()), IterableCoder.of(VarIntCoder.of()), PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
null, fn, NullSideInputReader.empty(), null, null,
private SplittableProcessElementInvoker<Void, String, OffsetRange, Long>.Result runTest( DoFn<Void, String> fn, OffsetRange initialRestriction) { SplittableProcessElementInvoker<Void, String, OffsetRange, Long> invoker = new OutputAndTimeBoundedSplittableProcessElementInvoker<>( fn, PipelineOptionsFactory.create(), new OutputWindowedValue<String>() { @Override public void outputWindowedValue( String output, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) {} @Override public <AdditionalOutputT> void outputWindowedValue( TupleTag<AdditionalOutputT> tag, AdditionalOutputT output, Instant timestamp, Collection<? extends BoundedWindow> windows, PaneInfo pane) {} }, NullSideInputReader.empty(), Executors.newSingleThreadScheduledExecutor(), 1000, Duration.standardSeconds(3)); return invoker.invokeProcessElement( DoFnInvokers.invokerFor(fn), WindowedValue.of(null, Instant.now(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING), new OffsetRangeTracker(initialRestriction)); }
null, fn, NullSideInputReader.empty(), null, null,
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, creating a {@link * TriggerStateMachine} from its {@link Trigger}. */ public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowingStrategy<?, W> windowingStrategy) throws Exception { return new ReduceFnTester<>( windowingStrategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger())), SystemReduceFn.buffering(VarIntCoder.of()), IterableCoder.of(VarIntCoder.of()), PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, {@link CombineFn}, * and {@link TriggerStateMachine}, for mocking the interaction between {@link ReduceFnRunner} and * the {@link TriggerStateMachine}. Ignores the {@link Trigger} in the {@link WindowingStrategy}. */ public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return new ReduceFnTester<>( strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
null, fn, NullSideInputReader.empty(), new ListOutputManager(), new TupleTag<>(),
null, fn, NullSideInputReader.empty(), new ListOutputManager(), new TupleTag<>(),