public static RunnerApi.Pipeline toProto(Pipeline pipeline, boolean useDeprecatedViewTransforms) { return toProto( pipeline, SdkComponents.create(pipeline.getOptions()), useDeprecatedViewTransforms); }
if (!useDeprecatedViewTransforms) { res = elideDeprecatedViews(res);
public static RunnerApi.Pipeline toProto(Pipeline pipeline) { return toProto(pipeline, SdkComponents.create(pipeline.getOptions())); }
/** Translates a {@link Pipeline} into a {@code JobSpecification}. */ public JobSpecification translate( Pipeline pipeline, DataflowRunner runner, List<DataflowPackage> packages) { // Capture the sdkComponents for look up during step translations SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.JAVA_SDK_HARNESS_ENVIRONMENT); RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, sdkComponents, true); LOG.debug("Portable pipeline proto:\n{}", TextFormat.printToString(pipelineProto)); Translator translator = new Translator(pipeline, runner, sdkComponents); Job result = translator.translate(packages); return new JobSpecification( result, pipelineProto, Collections.unmodifiableMap(translator.stepNames)); }
@Test public void testProtoDirectlyWithViewTransform() { final RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, true); pipeline.traverseTopologically(new PipelineProtoVerificationVisitor(pipelineProto, true)); }
@Test public void testProtoDirectly() { final RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline, false); pipeline.traverseTopologically(new PipelineProtoVerificationVisitor(pipelineProto, false)); }
PrepareJobRequest.newBuilder() .setJobName(options.getJobName()) .setPipeline(PipelineTranslation.toProto(pipeline)) .setPipelineOptions(PipelineOptionsTranslation.toProto(options)) .build();
@Test public void getEnvironmentWithEnvironment() { Pipeline p = Pipeline.create(); PCollection<Long> longs = p.apply("BoundedRead", Read.from(CountingSource.upTo(100L))); PCollectionList.of(longs).and(longs).and(longs).apply("flatten", Flatten.pCollections()); Components components = PipelineTranslation.toProto(p).getComponents(); QueryablePipeline qp = QueryablePipeline.forPrimitivesIn(components); PTransformNode environmentalRead = PipelineNode.pTransform("BoundedRead", components.getTransformsOrThrow("BoundedRead")); PTransformNode nonEnvironmentalTransform = PipelineNode.pTransform("flatten", components.getTransformsOrThrow("flatten")); assertThat(qp.getEnvironment(environmentalRead).isPresent(), is(true)); assertThat( qp.getEnvironment(environmentalRead).get(), equalTo(Environments.JAVA_SDK_HARNESS_ENVIRONMENT)); assertThat(qp.getEnvironment(nonEnvironmentalTransform).isPresent(), is(false)); }
@Test public void retainOnlyPrimitivesWithOnlyPrimitivesUnchanged() { Pipeline p = Pipeline.create(); p.apply("Read", Read.from(CountingSource.unbounded())) .apply( "multi-do", ParDo.of(new TestFn()).withOutputTags(new TupleTag<>(), TupleTagList.empty())); Components originalComponents = PipelineTranslation.toProto(p).getComponents(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(originalComponents); assertThat(primitiveComponents, equalTo(originalComponents.getTransformsMap().keySet())); }
@Test public void retainOnlyPrimitivesComposites() { Pipeline p = Pipeline.create(); p.apply( new org.apache.beam.sdk.transforms.PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return input .apply(GenerateSequence.from(2L)) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(5L)))) .apply(MapElements.into(TypeDescriptors.longs()).via(l -> l + 1)); } }); Components originalComponents = PipelineTranslation.toProto(p).getComponents(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(originalComponents); // Read, Window.Assign, ParDo. This will need to be updated if the expansions change. assertThat(primitiveComponents, hasSize(3)); for (String transformId : primitiveComponents) { assertThat(originalComponents.getTransformsMap(), hasKey(transformId)); } }
+ flinkCluster.getRestAddress().getPort()); RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
.apply(ParDo.of(collectResults)); RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p);
.withOutputTags(new TupleTag<>(), TupleTagList.empty())); Components components = PipelineTranslation.toProto(p).getComponents(); QueryablePipeline qp = QueryablePipeline.forPrimitivesIn(components);
/** * Tests that {@link QueryablePipeline#getPerElementConsumers(PCollectionNode)} returns a * transform that consumes the node more than once. */ @Test public void perElementConsumersWithConsumingMultipleTimes() { Pipeline p = Pipeline.create(); PCollection<Long> longs = p.apply("BoundedRead", Read.from(CountingSource.upTo(100L))); PCollectionList.of(longs).and(longs).and(longs).apply("flatten", Flatten.pCollections()); Components components = PipelineTranslation.toProto(p).getComponents(); // This breaks if the way that IDs are assigned to PTransforms changes in PipelineTranslation String readOutput = getOnlyElement(components.getTransformsOrThrow("BoundedRead").getOutputsMap().values()); QueryablePipeline qp = QueryablePipeline.forPrimitivesIn(components); Set<PTransformNode> consumers = qp.getPerElementConsumers( PipelineNode.pCollection(readOutput, components.getPcollectionsOrThrow(readOutput))); assertThat(consumers.size(), equalTo(1)); assertThat( getOnlyElement(consumers).getTransform().getSpec().getUrn(), equalTo(PTransformTranslation.FLATTEN_TRANSFORM_URN)); }
RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
@Test public void rootTransforms() { Pipeline p = Pipeline.create(); p.apply("UnboundedRead", Read.from(CountingSource.unbounded())) .apply(Window.into(FixedWindows.of(Duration.millis(5L)))) .apply(Count.perElement()); p.apply("BoundedRead", Read.from(CountingSource.upTo(100L))); Components components = PipelineTranslation.toProto(p).getComponents(); QueryablePipeline qp = QueryablePipeline.forPrimitivesIn(components); assertThat(qp.getRootTransforms(), hasSize(2)); for (PTransformNode rootTransform : qp.getRootTransforms()) { assertThat( "Root transforms should have no inputs", rootTransform.getTransform().getInputsCount(), equalTo(0)); assertThat( "Only added source reads to the pipeline", rootTransform.getTransform().getSpec().getUrn(), equalTo(PTransformTranslation.READ_TRANSFORM_URN)); } }
RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(p);
PCollectionList.of(longs).and(longs).and(longs).apply("flatten", Flatten.pCollections()); Components components = PipelineTranslation.toProto(p).getComponents(); QueryablePipeline qp = QueryablePipeline.forPrimitivesIn(components);
/** This method doesn't do any pruning for reachability, but this may not require a test. */ @Test public void retainOnlyPrimitivesIgnoresUnreachableNodes() { Pipeline p = Pipeline.create(); p.apply( new org.apache.beam.sdk.transforms.PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return input .apply(GenerateSequence.from(2L)) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(5L)))) .apply(MapElements.into(TypeDescriptors.longs()).via(l -> l + 1)); } }); Components augmentedComponents = PipelineTranslation.toProto(p) .getComponents() .toBuilder() .putCoders("extra-coder", RunnerApi.Coder.getDefaultInstance()) .putWindowingStrategies( "extra-windowing-strategy", RunnerApi.WindowingStrategy.getDefaultInstance()) .putEnvironments("extra-env", RunnerApi.Environment.getDefaultInstance()) .putPcollections("extra-pc", RunnerApi.PCollection.getDefaultInstance()) .build(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(augmentedComponents); } }