public static PTransformOverride boundedOverride() { return PTransformOverride.of(boundedMatcher(), new BoundedOverrideFactory<>()); }
@Override public void visitPrimitiveTransform(Node node) { if (freedNodes.contains(node.getEnclosingNode())) { freedNodes.add(node); } else if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) { matches.add(node); freedNodes.add(node); } } });
applyReplacement(match, override.getOverrideFactory());
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner builder.add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Spark runner currently doesn't support @RequiresStableInput annotation."))); if (!streaming) { builder .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), new SplittableParDoNaiveBounded.OverrideFactory())); } return builder.build(); } }
@Override public CompositeBehavior enterCompositeTransform(Node node) { if (!node.isRootNode() && freedNodes.contains(node.getEnclosingNode())) { // This node will be freed because its parent will be freed. freedNodes.add(node); return CompositeBehavior.ENTER_TRANSFORM; } if (!node.isRootNode() && override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) { matches.add(node); // This node will be freed. When we visit any of its children, they will also be freed freedNodes.add(node); } return CompositeBehavior.ENTER_TRANSFORM; }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); builder // TODO: [BEAM-5359] Support @RequiresStableInput on Flink runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), streaming ? new SplittableParDoViaKeyedWorkItems.OverrideFactory() : new SplittableParDoNaiveBounded.OverrideFactory())); if (streaming) { builder.add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory())); } return builder.build(); } }
private void checkForMatches(Node node) { for (PTransformOverride override : overrides) { if (override .getMatcher() .matchesDuringValidation(node.toAppliedPTransform(getPipeline()))) { matched.put(node, override); } } } });
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { if (streaming) { return ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new FlinkStreamingPipelineTranslator.SplittableParDoOverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.classEqualTo(SplittableParDo.ProcessKeyedElements.class), new SplittableParDoViaKeyedWorkItems.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingFlinkView.Factory())) .build(); } else { return ImmutableList.of(); } } }
return ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.classEqualTo(Create.Values.class), new PrimitiveCreate.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.IterableViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.ListViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.MapViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.MultimapViewFn.class), new StreamingViewAsIterable.Factory())) .add( PTransformOverride.of( PTransformMatchers.createViewWithViewFn(PCollectionViews.SingletonViewFn.class), new StreamingWrapSingletonInList.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new SplittableParDo.OverrideFactory())) .add(
public void translate(Pipeline pipeline) { List<PTransformOverride> overrides = ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingGearpumpView.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.splittableProcessKeyedBounded(), new SplittableParDoNaiveBounded.OverrideFactory())) // TODO: [BEAM-5361] Support @RequiresStableInput on Gearpump runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Gearpump runner currently doesn't support @RequiresStableInput annotation."))) .build(); pipeline.replaceAll(overrides); pipeline.traverseTopologically(this); }
PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), options.isStreaming() builder .add( PTransformOverride.of( PTransformMatchers.writeWithRunnerDeterminedSharding(), new FlinkStreamingPipelineTranslator.StreamingShardedWriteFactory( checkNotNull(options)))) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory()));
if (testOptions.isRunnerDeterminedSharding()) { builder.add( PTransformOverride.of( PTransformMatchers.writeWithRunnerDeterminedSharding(), new WriteWithShardingFactory())); /* Uses a view internally. */ builder .add( PTransformOverride.of( MultiStepCombine.matcher(), MultiStepCombine.Factory.create())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new ViewOverrideFactory())) /* Uses pardos and GBKs */ .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.TEST_STREAM_TRANSFORM_URN), new DirectTestStreamFactory(this))) /* primitive */ PTransformOverride.of( PTransformMatchers.splittableParDo(), new ParDoMultiOverrideFactory())) PTransformOverride.of( PTransformMatchers.stateOrTimerParDo(), new ParDoMultiOverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo( PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), new SplittableParDoViaKeyedWorkItems.OverrideFactory()))
PTransformOverride.of( PTransformMatchers.flattenWithDuplicateInputs(), DeduplicatedFlattenFactory.create())) .add( PTransformOverride.of( PTransformMatchers.emptyFlatten(), EmptyFlattenAsCreateFactory.instance())) PTransformOverride.of( PTransformMatchers.splittableParDoSingle(), new ReflectiveOneToOneOverrideFactory( SplittableParDoOverrides.ParDoSingleViaMulti.class, this))) .add( PTransformOverride.of( PTransformMatchers.splittableParDoMulti(), new SplittableParDoOverrides.SplittableParDoOverrideFactory())); if (!hasExperiment(options, "enable_custom_pubsub_source")) { overridesBuilder.add( PTransformOverride.of( PTransformMatchers.classEqualTo(PubsubUnboundedSource.class), new StreamingPubsubIOReadOverrideFactory())); PTransformOverride.of( PTransformMatchers.classEqualTo(PubsubUnboundedSink.class), new StreamingPubsubIOWriteOverrideFactory(this))); PTransformOverride.of( PTransformMatchers.classEqualTo(Create.Values.class), new StreamingFnApiCreateOverrideFactory()));
pipeline.replaceAll( Collections.singletonList( PTransformOverride.of(new OriginalMatcher(), new ReplacementOverrideFactory()))); final Map<String, Class<?>> nameToTransformClass = new HashMap<>(); pipeline.traverseTopologically(
/** * Tests that {@link Pipeline#replaceAll(List)} throws when one of the PTransformOverride still * matches. */ @Test public void testReplaceAllIncomplete() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.apply(GenerateSequence.from(0)); // The order is such that the output of the second will match the first, which is not permitted thrown.expect(IllegalStateException.class); pipeline.replaceAll( ImmutableList.of( PTransformOverride.of( application -> application.getTransform() instanceof Create.Values, new CreateValuesToEmptyFlattenOverride()), PTransformOverride.of( application -> application.getTransform() instanceof GenerateSequence, new GenerateSequenceToCreateOverride()))); }
@Test public void testReplaceAll() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.apply("unbounded", GenerateSequence.from(0)); pipeline.apply("bounded", GenerateSequence.from(0).to(100)); pipeline.replaceAll( ImmutableList.of( PTransformOverride.of( application -> application.getTransform() instanceof GenerateSequence, new GenerateSequenceToCreateOverride()), PTransformOverride.of( application -> application.getTransform() instanceof Create.Values, new CreateValuesToEmptyFlattenOverride()))); pipeline.traverseTopologically( new PipelineVisitor.Defaults() { @Override public CompositeBehavior enterCompositeTransform(Node node) { if (!node.isRootNode()) { assertThat( node.getTransform().getClass(), not( anyOf( Matchers.equalTo(GenerateSequence.class), Matchers.equalTo(Create.Values.class)))); } return CompositeBehavior.ENTER_TRANSFORM; } }); }
Arrays.asList( JavaReadViaImpulse.boundedOverride(), PTransformOverride.of( PTransformMatchers.splittableParDo(), new ParDoMultiOverrideFactory())));