.add( PTransformOverride.of( PTransformMatchers.classEqualTo(Create.Values.class), new PrimitiveCreate.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.ListViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.MapViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.MultimapViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class), new StreamingWrapSingletonInList.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of(
public void translate(Pipeline pipeline) { List<PTransformOverride> overrides = ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingGearpumpView.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.splittableProcessKeyedBounded(), new SplittableParDoNaiveBounded.OverrideFactory())) // TODO: [BEAM-5361] Support @RequiresStableInput on Gearpump runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Gearpump runner currently doesn't support @RequiresStableInput annotation."))) .build(); pipeline.replaceAll(overrides); pipeline.traverseTopologically(this); }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner builder.add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Spark runner currently doesn't support @RequiresStableInput annotation."))); if (!streaming) { builder .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), new SplittableParDoNaiveBounded.OverrideFactory())); } return builder.build(); } }
@Test public void parDoSingleWithState() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform(ParDo.of(doFnWithState)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { if (streaming) { return ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new FlinkStreamingPipelineTranslator.SplittableParDoOverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class), new SplittableParDoViaKeyedWorkItems.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingFlinkView.Factory())) .build(); } else { return ImmutableList.of(); } } }
.add( PTransformOverride.of( PTransformMatchers.flattenWithDuplicateInputs(), DeduplicatedFlattenFactory.create())) .add( PTransformOverride.of( PTransformMatchers.emptyFlatten(), EmptyFlattenAsCreateFactory.instance())) PTransformMatchers.splittableParDoSingle(), new ReflectiveOneToOneOverrideFactory( SplittableParDoOverrides.ParDoSingleViaMulti.class, this))) .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new SplittableParDoOverrides.SplittableParDoOverrideFactory())); if (streaming) { overridesBuilder.add( PTransformOverride.of( PTransformMatchers.classEqualTo(PubsubUnboundedSource.class), new StreamingPubsubIOReadOverrideFactory())); overridesBuilder.add( PTransformOverride.of( PTransformMatchers.classEqualTo(PubsubUnboundedSink.class), new StreamingPubsubIOWriteOverrideFactory(this))); overridesBuilder.add( PTransformOverride.of(
PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), options.isStreaming() ? new SplittableParDoViaKeyedWorkItems.OverrideFactory() .add( PTransformOverride.of( PTransformMatchers.writeWithRunnerDeterminedSharding(), new FlinkStreamingPipelineTranslator.StreamingShardedWriteFactory( checkNotNull(options)))) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory()));
@Test public void parDoSplittable() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDo().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoRequiresStableInput() { DoFn<Object, Object> doFnRSI = new DoFn<Object, Object>() { @RequiresStableInput @ProcessElement public void process(ProcessContext ctxt) {} }; AppliedPTransform<?, ?, ?> single = getAppliedTransform(ParDo.of(doFn)); AppliedPTransform<?, ?, ?> singleRSI = getAppliedTransform(ParDo.of(doFnRSI)); AppliedPTransform<?, ?, ?> multi = getAppliedTransform(ParDo.of(doFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); AppliedPTransform<?, ?, ?> multiRSI = getAppliedTransform( ParDo.of(doFnRSI).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(single), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(singleRSI), is(true)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(multi), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(multiRSI), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(single), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(singleRSI), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(multi), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(multiRSI), is(true)); }
@Test public void classEqualToDoesNotMatchSubclass() { class MyPTransform extends PTransform<PCollection<KV<String, Integer>>, PCollection<Integer>> { @Override public PCollection<Integer> expand(PCollection<KV<String, Integer>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), VarIntCoder.of()); } } PTransformMatcher matcher = PTransformMatchers.classEqualTo(MyPTransform.class); MyPTransform subclass = new MyPTransform() {}; assertThat(subclass.getClass(), not(Matchers.<Class<?>>equalTo(MyPTransform.class))); assertThat(subclass, instanceOf(MyPTransform.class)); AppliedPTransform<?, ?, ?> application = getAppliedTransform(subclass); assertThat(matcher.matches(application), is(false)); }
private static PTransformMatcher boundedMatcher() { return PTransformMatchers.urnEqualTo(PTransformTranslation.READ_TRANSFORM_URN) .and( transform -> ReadTranslation.sourceIsBounded(transform) == PCollection.IsBounded.BOUNDED); }
@Test public void createViewWithViewFnDifferentViewFn() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); // Purposely create a subclass to get a different class then what was expected. ViewFn<?, ?> viewFn = new PCollectionViews.IterableViewFn() {}; CreatePCollectionView<?, ?> createView = CreatePCollectionView.of(view); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(viewFn.getClass()); assertThat(matcher.matches(getAppliedTransform(createView)), is(false)); }
PTransformMatchers.writeWithRunnerDeterminedSharding().matches(appliedWrite(write)), is(true)); PTransformMatchers.writeWithRunnerDeterminedSharding() .matches(appliedWrite(withStaticSharding)), is(false)); write.withSharding(Sum.integersGlobally().asSingletonView()); assertThat( PTransformMatchers.writeWithRunnerDeterminedSharding() .matches(appliedWrite(withCustomSharding)), is(false));
/** Demonstrates that a {@link ParDo.SingleOutput} does not match any ParDo matcher. */ @Test public void parDoSingle() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform(ParDo.of(doFn)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void classEqualToMatchesSameClass() { PTransformMatcher matcher = PTransformMatchers.classEqualTo(ParDo.SingleOutput.class); AppliedPTransform<?, ?, ?> application = getAppliedTransform( ParDo.of( new DoFn<KV<String, Integer>, Integer>() { @ProcessElement public void doStuff(ProcessContext ctxt) {} })); assertThat(matcher.matches(application), is(true)); }
@Test public void createViewWithViewFn() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); ViewFn<?, ?> viewFn = view.getViewFn(); CreatePCollectionView<?, ?> createView = CreatePCollectionView.of(view); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(viewFn.getClass()); assertThat(matcher.matches(getAppliedTransform(createView)), is(true)); }
@Test public void parDoSingleSplittable() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform(ParDo.of(splittableDoFn)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); builder // TODO: [BEAM-5359] Support @RequiresStableInput on Flink runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), streaming ? new SplittableParDoViaKeyedWorkItems.OverrideFactory() : new SplittableParDoNaiveBounded.OverrideFactory())); if (streaming) { builder.add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory())); } return builder.build(); } }
@Test public void classEqualToDoesNotMatchUnrelatedClass() { PTransformMatcher matcher = PTransformMatchers.classEqualTo(ParDo.SingleOutput.class); AppliedPTransform<?, ?, ?> application = getAppliedTransform(Window.<KV<String, Integer>>into(new GlobalWindows())); assertThat(matcher.matches(application), is(false)); }
@Test public void createViewWithViewFnNotCreatePCollectionView() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(view.getViewFn().getClass()); assertThat(matcher.matches(getAppliedTransform(View.asIterable())), is(false)); }