@Test public void parDoRequiresStableInput() { DoFn<Object, Object> doFnRSI = new DoFn<Object, Object>() { @RequiresStableInput @ProcessElement public void process(ProcessContext ctxt) {} }; AppliedPTransform<?, ?, ?> single = getAppliedTransform(ParDo.of(doFn)); AppliedPTransform<?, ?, ?> singleRSI = getAppliedTransform(ParDo.of(doFnRSI)); AppliedPTransform<?, ?, ?> multi = getAppliedTransform(ParDo.of(doFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); AppliedPTransform<?, ?, ?> multiRSI = getAppliedTransform( ParDo.of(doFnRSI).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(single), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(singleRSI), is(true)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(multi), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoSingle().matches(multiRSI), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(single), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(singleRSI), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(multi), is(false)); assertThat(PTransformMatchers.requiresStableInputParDoMulti().matches(multiRSI), is(true)); }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner builder.add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Spark runner currently doesn't support @RequiresStableInput annotation."))); if (!streaming) { builder .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), new SplittableParDoNaiveBounded.OverrideFactory())); } return builder.build(); } }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); builder // TODO: [BEAM-5359] Support @RequiresStableInput on Flink runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), streaming ? new SplittableParDoViaKeyedWorkItems.OverrideFactory() : new SplittableParDoNaiveBounded.OverrideFactory())); if (streaming) { builder.add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory())); } return builder.build(); } }
public void translate(Pipeline pipeline) { List<PTransformOverride> overrides = ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingGearpumpView.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.splittableProcessKeyedBounded(), new SplittableParDoNaiveBounded.OverrideFactory())) // TODO: [BEAM-5361] Support @RequiresStableInput on Gearpump runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Gearpump runner currently doesn't support @RequiresStableInput annotation."))) .build(); pipeline.replaceAll(overrides); pipeline.traverseTopologically(this); }
PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation.")))
PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Apex runner currently doesn't support @RequiresStableInput annotation.")))
PTransformMatchers.requiresStableInputParDoMulti(), RequiresStableInputParDoOverrides.multiOutputOverrideFactory()));