.getComponents() .getPcollectionsOrThrow(inputPCollectionId) .getWindowingStrategyId());
/** Indicates whether the given pipeline has any unbounded PCollections. */ private static boolean hasUnboundedPCollections(RunnerApi.Pipeline pipeline) { checkNotNull(pipeline); Collection<RunnerApi.PCollection> pCollecctions = pipeline.getComponents().getPcollectionsMap().values(); // Assume that all PCollections are consumed at some point in the pipeline. return pCollecctions .stream() .anyMatch(pc -> pc.getIsBounded() == RunnerApi.IsBounded.Enum.UNBOUNDED); } }
PCollection pc = components.getPcollectionsOrThrow(pcollectionId); checkArgument( !pc.getUniqueName().isEmpty(), "%s: PCollection %s does not have a unique_name set", context, pcollectionId); String previousId = uniqueNamesById.put(pc.getUniqueName(), pcollectionId); checkArgument( previousId == null, pcollectionId, previousId, pc.getUniqueName()); checkArgument( components.containsCoders(pc.getCoderId()), "%s: PCollection %s uses unknown coder %s", context, pcollectionId, pc.getCoderId()); checkArgument( components.containsWindowingStrategies(pc.getWindowingStrategyId()), "%s: PCollection %s uses unknown windowing strategy %s", context, pcollectionId, pc.getWindowingStrategyId());
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection threeOut = PCollection.newBuilder().setUniqueName("three.out").build(); PTransform three = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", threeOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder()
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder()
.build(); PCollection flattenPc = PCollection.newBuilder().setUniqueName("flatten.out").build(); Components components = partialComponents .putTransforms("pyRead", pyRead) .putPcollections( "pyRead.out", PCollection.newBuilder().setUniqueName("pyRead.out").build()) .putTransforms("goRead", goRead) .putPcollections( "goRead.out", PCollection.newBuilder().setUniqueName("goRead.out").build()) .putTransforms( "flatten", .putTransforms("pyParDo", pyParDo) .putPcollections( "pyParDo.out", PCollection.newBuilder().setUniqueName("pyParDo.out").build()) .putTransforms("goWindow", goWindow) .putPcollections( "goWindow.out", PCollection.newBuilder().setUniqueName("goWindow.out").build()) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py"))
Coder inputCoder = components.getCodersOrThrow(input.getCoderId()); KvCoderComponents kvComponents = ModelCoders.getKvCoderComponents(inputCoder); String windowCoderId = components .getWindowingStrategiesOrThrow(input.getWindowingStrategyId()) .getWindowCoderId(); input.toBuilder().setUniqueName(kwiCollectionId).setCoderId(kwiCoderId).build(); String rawGbkId = uniqueId(String.format("%s/RawGBK", spkId), components::containsTransforms); input.toBuilder().setUniqueName(feedSDFCollectionId).setCoderId(feedSDFCoderId).build(); String feedSDFId = uniqueId(String.format("%s/FeedSDF", spkId), components::containsTransforms);
.toBuilder() .putTransforms("read", readTransform) .putPcollections("read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms("envRead", otherEnvRead) .putPcollections( "envRead.out", PCollection.newBuilder().setUniqueName("envRead.out").build()) .putTransforms("flatten", flattenTransform) .putPcollections( "flatten.out", PCollection.newBuilder().setUniqueName("flatten.out").build()) .putTransforms("window", windowTransform) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .putEnvironments("rare", Environments.createDockerEnvironment("rare"))
PTransform.newBuilder() .setUniqueName("Flatten") .putInputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .putInputs(read2Output.getUniqueName(), read2Output.getUniqueName()) .putOutputs(flattenOutput.getUniqueName(), flattenOutput.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("read1") .putInputs(impulse1Output.getUniqueName(), impulse1Output.getUniqueName()) .putOutputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("read2") .putInputs(impulse2Output.getUniqueName(), impulse2Output.getUniqueName()) .putOutputs(read2Output.getUniqueName(), read2Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("impulse1") .putOutputs(impulse1Output.getUniqueName(), impulse1Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder()
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("red", redOut.getUniqueName()) .putOutputs("shared", sharedOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); PTransform blue =
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); PTransform blue = PTransform.newBuilder()
PipelineNode.pCollection( "values", RunnerApi.PCollection.newBuilder() .setUniqueName("values") .setCoderId(coderId) PCollectionNode groupedKvs = PipelineNode.pCollection( "groupedKvs", RunnerApi.PCollection.newBuilder().setUniqueName("groupedKvs").build()); PTransformNode groupByKeyOnly = PipelineNode.pTransform(
.build(); PCollection sideInputPCollection = PCollection.newBuilder().setUniqueName("side_read.out").build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn( .putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", .putPcollections( "side_read.out", PCollection.newBuilder().setUniqueName("side_read.out").build()) .putTransforms( "parDo", .build()) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms( "window", .build()) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", env) .build());
.toByteString())) .build(); PCollection input = PCollection.newBuilder().setUniqueName("input.out").build(); PCollection sideInput = PCollection.newBuilder().setUniqueName("sideInput.in").build(); PCollection timer = PCollection.newBuilder().setUniqueName("timer.out").build(); PCollection output = PCollection.newBuilder().setUniqueName("output.out").build();
.putPcollections( "intermediatePc", PCollection.newBuilder().setUniqueName("intermediate").build()) .putCoders( "coder", .putPcollections( "intermediatePc", PCollection.newBuilder().setUniqueName("intermediate_replacement").build()) .putWindowingStrategies("new_ws", introducedWS) .putTransforms("second_sub", PTransform.getDefaultInstance()) updated.getComponents().getPcollectionsOrThrow("intermediatePc").getUniqueName(), equalTo("intermediate_replacement"));
.getComponents() .getPcollectionsOrThrow(inputPCollectionId) .getWindowingStrategyId());
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms("flatten", flattenTransform) .putPcollections( "flatten.out", PCollection.newBuilder().setUniqueName("flatten.out").build()) .putTransforms("window", windowTransform) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.build(); PCollection sidePC = PCollection.newBuilder().setUniqueName("sidePC").build(); PCollection impulsePC = PCollection.newBuilder().setUniqueName("impulsePC").build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn(
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "parDo", .build()) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms( "window", .build()) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("rare", Environments.createDockerEnvironment("rare")) .putEnvironments("common", env)