.putPcollections("timer.pc", timer) .putPcollections("output.out", output) .putEnvironments("foo", env) .build();
.putPcollections("timer.out", timer) .putPcollections("output.out", output) .putEnvironments("foo", env) .build();
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn(components);
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", env) .build());
.putPcollections("impulsePC", impulsePC) .putPcollections("sidePC", sidePC) .putEnvironments("common", env) .build());
.putTransforms("stateful", statefulTransform) .putPcollections("stateful.out", pc("stateful.out")) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build(); FusedPipeline fused =
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("rare", Environments.createDockerEnvironment("rare")) .putEnvironments("common", env) .build());
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.putPcollections( "stateful.out", PCollection.newBuilder().setUniqueName("stateful.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.putPcollections("timer.out", pc("timer.out")) .putPcollections("output.out", pc("output.out")) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build();
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("rare", Environments.createDockerEnvironment("rare")) .putEnvironments("common", env) .build());
.putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
.putPcollections( "timer.out", PCollection.newBuilder().setUniqueName("timer.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.build()) .putPcollections("py.out", PCollection.newBuilder().setUniqueName("py.out").build()) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .build()); Set<PTransformNode> differentEnvironments =
.putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.putPcollections("timer.out", pc("timer.out")) .putPcollections("output.out", pc("output.out")) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build();
.putPcollections( "gbk.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
/** This method doesn't do any pruning for reachability, but this may not require a test. */ @Test public void retainOnlyPrimitivesIgnoresUnreachableNodes() { Pipeline p = Pipeline.create(); p.apply( new org.apache.beam.sdk.transforms.PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return input .apply(GenerateSequence.from(2L)) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(5L)))) .apply(MapElements.into(TypeDescriptors.longs()).via(l -> l + 1)); } }); Components augmentedComponents = PipelineTranslation.toProto(p) .getComponents() .toBuilder() .putCoders("extra-coder", RunnerApi.Coder.getDefaultInstance()) .putWindowingStrategies( "extra-windowing-strategy", RunnerApi.WindowingStrategy.getDefaultInstance()) .putEnvironments("extra-env", RunnerApi.Environment.getDefaultInstance()) .putPcollections("extra-pc", RunnerApi.PCollection.getDefaultInstance()) .build(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(augmentedComponents); } }
@Before public void setup() { partialComponents = Components.newBuilder() .putTransforms( "impulse", PTransform.newBuilder() .setUniqueName("Impulse") .putOutputs("output", "impulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("impulse.out", pc("impulse.out")) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .build(); }
/** * Registers the provided {@link Environment} into this {@link SdkComponents}, returning a unique * ID for the {@link Environment}. Multiple registrations of the same {@link Environment} will * return the same unique ID. */ public String registerEnvironment(Environment env) { String existing = environmentIds.get(env); if (existing != null) { return existing; } String name = uniqify(env.getUrn(), environmentIds.values()); environmentIds.put(env, name); componentsBuilder.putEnvironments(name, env); return name; }