SystemReduceFn.buffering(this.valueCoder), serializedOptions.get());
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFnWithContext<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder, PipelineOptions options, SideInputReader sideInputReader) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return new ReduceFnTester<>( strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, options, sideInputReader); }
@SafeVarargs public final void assertHasOnlyGlobalAndStateFor(W... expectedWindows) { assertHasOnlyGlobalAndAllowedTags( ImmutableSet.copyOf(expectedWindows), ImmutableSet.of( ((SystemReduceFn<?, ?, ?, ?, ?>) reduceFn).getBufferTag(), TriggerStateMachineRunner.FINISHED_BITS_TAG, PaneInfoTracker.PANE_INFO_TAG, WatermarkHold.watermarkHoldTagForTimestampCombiner( objectStrategy.getTimestampCombiner()), WatermarkHold.EXTRA_HOLD_TAG)); }
/** * Create a group by key transform. * It returns GroupByKeyAndWindowDoFnTransform if window function is not default. * @param ctx translation context * @param beamNode the beam node to be translated * @return group by key transform */ private static Transform createGBKTransform( final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode) { final AppliedPTransform pTransform = beamNode.toAppliedPTransform(ctx.getPipeline()); final PCollection<?> mainInput = (PCollection<?>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform)); final TupleTag mainOutputTag = new TupleTag<>(); if (isGlobalWindow(beamNode, ctx.getPipeline())) { return new GroupByKeyTransform(); } else { return new GroupByKeyAndWindowDoFnTransform( getOutputCoders(pTransform), mainOutputTag, mainInput.getWindowingStrategy(), ctx.getPipelineOptions(), SystemReduceFn.buffering(mainInput.getCoder()), DisplayData.from(beamNode.getTransform())); } }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, {@link CombineFn}, * and {@link TriggerStateMachine}, for mocking the interaction between {@link ReduceFnRunner} and * the {@link TriggerStateMachine}. Ignores the {@link Trigger} in the {@link WindowingStrategy}. */ public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return new ReduceFnTester<>( strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
SystemReduceFn.buffering(wvCoder.getValueCoder());
SystemReduceFn.combining( inputKvCoder.getKeyCoder(), AppliedCombineFn.withInputCoder(
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link * TriggerStateMachine}, for mocking the interactions between {@link ReduceFnRunner} and the * {@link TriggerStateMachine}. * * <p>Ignores the {@link Trigger} on the {@link WindowingStrategy}. */ public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowingStrategy<?, W> windowingStrategy, TriggerStateMachine triggerStateMachine) throws Exception { return new ReduceFnTester<>( windowingStrategy, triggerStateMachine, SystemReduceFn.buffering(VarIntCoder.of()), IterableCoder.of(VarIntCoder.of()), PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
SystemReduceFn.combining( inputKvCoder.getKeyCoder(), AppliedCombineFn.withInputCoder(
SystemReduceFn.buffering(inputElementCoder.getValueCoder());
SystemReduceFn.combining( VarLongCoder.of(), AppliedCombineFn.withInputCoder(
SystemReduceFn.buffering(inputElementCoder.getValueCoder());
.keyBy(new WorkItemKeySelector<K, InputT>(inputKvCoder.getKeyCoder())); SystemReduceFn<K, InputT, ?, OutputT, BoundedWindow> reduceFn = SystemReduceFn.combining( inputKvCoder.getKeyCoder(), AppliedCombineFn.withInputCoder(
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, creating a {@link * TriggerStateMachine} from its {@link Trigger}. */ public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowingStrategy<?, W> windowingStrategy) throws Exception { return new ReduceFnTester<>( windowingStrategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger())), SystemReduceFn.buffering(VarIntCoder.of()), IterableCoder.of(VarIntCoder.of()), PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
SystemReduceFn.combining( VarLongCoder.of(), AppliedCombineFn.withInputCoder(
SystemReduceFn.buffering(inputKvCoder.getValueCoder());
SystemReduceFn.buffering(inputKvCoder.getValueCoder());
SystemReduceFn.buffering(inputKvCoder.getValueCoder());