PTransform red = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("red", redOut.getUniqueName()) .putOutputs("shared", sharedOut.getUniqueName()) .build();
PTransform rawGbk = PTransform.newBuilder() .setUniqueName(String.format("%s/RawGBK", spk.getUniqueName())) .putAllInputs(spk.getInputsMap()) .setSpec(FunctionSpec.newBuilder().setUrn(DirectGroupByKey.DIRECT_GBKO_URN)) .putOutputs("output", kwiCollectionId) .build(); .putPcollections(kwiCollectionId, kwiCollection) .putTransforms(rawGbkId, rawGbk); newPTransform.addSubtransforms(rawGbkId); PTransform feedSDF = PTransform.newBuilder() .setUniqueName(String.format("%s/FeedSDF", spk.getUniqueName())) .putInputs("input", kwiCollectionId) .setSpec( FunctionSpec.newBuilder() .setUrn(SplittableRemoteStageEvaluatorFactory.FEED_SDF_URN)) .putOutputs("output", feedSDFCollectionId) .build(); newPTransform.addSubtransforms(feedSDFId); PTransform runSDF = PTransform.newBuilder() .setUniqueName(String.format("%s/RunSDF", spk.getUniqueName())) .putInputs("input", feedSDFCollectionId) .setSpec(
(key, value) -> { if (viewOutputsToInputs.containsKey(value)) { transformBuilder.putInputs(key, viewOutputsToInputs.get(value)); (key, value) -> { if (viewOutputsToInputs.containsKey(value)) { transformBuilder.putOutputs(key, viewOutputsToInputs.get(value)); transformBuilder.clearSubtransforms(); transformBuilder.addAllSubtransforms( transform .getSubtransformsList() .filter(id -> !viewTransforms.contains(id)) .collect(Collectors.toList())); newTransforms.put(transformId, transformBuilder.build()); });
PTransform red = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection threeOut = PCollection.newBuilder().setUniqueName("three.out").build(); PTransform three = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", threeOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build();
"mainImpulse", PTransform.newBuilder() .setUniqueName("MainImpulse") .putOutputs("output", "mainImpulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("mainImpulse.out", pc("mainImpulse.out")) .putTransforms( "read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "mainImpulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "sideImpulse", PTransform.newBuilder() .setUniqueName("SideImpulse") .putOutputs("output", "sideImpulse.out") .setSpec(
"pyImpulse", PTransform.newBuilder() .setUniqueName("PyImpulse") .putOutputs("output", "pyImpulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .putTransforms( "pyRead", PTransform.newBuilder() .setUniqueName("PyRead") .putInputs("input", "pyImpulse.out") .putOutputs("output", "pyRead.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .putTransforms( "goImpulse", PTransform.newBuilder() .setUniqueName("GoImpulse") .putOutputs("output", "goImpulse.out") .setSpec(
.putInputs("input", "impulse.out") .putOutputs("output", "pyRead.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .toByteString()) .build()) .build(); PTransform goRead = PTransform.newBuilder() .putInputs("input", "impulse.out") .putOutputs("output", "goRead.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .toByteString()) .build()) .build(); .putInputs("input", "flatten.out") .putOutputs("output", "pyParDo.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .toByteString()) .build())
PTransform red = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build();
"pyImpulse", PTransform.newBuilder() .setUniqueName("PyImpulse") .putOutputs("output", "pyImpulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .putTransforms( "pyRead", PTransform.newBuilder() .setUniqueName("PyRead") .putInputs("input", "pyImpulse.out") .putOutputs("output", "pyRead.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .putTransforms( "goImpulse", PTransform.newBuilder() .setUniqueName("GoImpulse") .putOutputs("output", "goImpulse.out") .setSpec(
.putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); PTransform otherEnvRead = PTransform.newBuilder() .putInputs("impulse", "impulse.out") .putOutputs("output", "envRead.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); PTransform flattenTransform = PTransform.newBuilder() .putInputs("readInput", "read.out") .putInputs("otherEnvInput", "envRead.out") .putOutputs("output", "flatten.out") .setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN)) .build(); PTransform windowTransform = PTransform.newBuilder()
PTransform flattenTransform = PTransform.newBuilder() .setUniqueName("Flatten") .putInputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .putInputs(read2Output.getUniqueName(), read2Output.getUniqueName()) .putOutputs(flattenOutput.getUniqueName(), flattenOutput.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN) .build() .toByteString())) .build(); .setUniqueName("read1") .putInputs(impulse1Output.getUniqueName(), impulse1Output.getUniqueName()) .putOutputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); PTransform read2Transform = PTransform.newBuilder() .setUniqueName("read2") .putInputs(impulse2Output.getUniqueName(), impulse2Output.getUniqueName()) .putOutputs(read2Output.getUniqueName(), read2Output.getUniqueName())
"source", PTransform.newBuilder().putOutputs("out", "values").build()); PCollectionNode values = PipelineNode.pCollection( "gbko", PTransform.newBuilder() .putInputs("input", "values") .putOutputs("output", "groupedKvs") .setSpec(FunctionSpec.newBuilder().setUrn(DirectGroupByKey.DIRECT_GBKO_URN).build()) .build()); Pipeline pipeline = Pipeline.newBuilder()
PTransform red = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build();
PipelineNode.pTransform( transformNode.getId(), transform.toBuilder().clearInputs().putAllInputs(validInputs).build());
PTransform readTransform = PTransform.newBuilder() .putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); "side_read", PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)) .putInputs("input", "impulse.out") .putOutputs("output", "side_read.out") .build()) .putPcollections( "side_read.out", "parDo", PTransform.newBuilder() .putInputs("input", "read.out") .putInputs("side_input", "side_read.out") .putOutputs("output", "parDo.out") .setSpec(
PTransform readTransform = PTransform.newBuilder() .setUniqueName("read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); .setUniqueName("parDo") .putInputs("input", "read.out") .putInputs("side_input", "side_read.out") .putOutputs("output", "parDo.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); PCollection sideInputPCollection = PCollection.newBuilder().setUniqueName("side_read.out").build(); "side_read", PTransform.newBuilder()
"first", PTransform.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn("beam:first")) .build()) .putTransforms( "second", PTransform.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn("beam:second")) .build()) .putPcollections( "intermediatePc", .addSubtransforms("second_sub") .setSpec( FunctionSpec.newBuilder() .setUrn("beam:second:replacement") .setPayload( ByteString.copyFrom("foo-bar-baz".getBytes(StandardCharsets.UTF_8)))) .build(); WindowingStrategy introducedWS = WindowingStrategy.newBuilder().setAccumulationMode(Enum.ACCUMULATING).build();
"read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "goTransform", PTransform.newBuilder() .setUniqueName("GoTransform") .putInputs("input", "read.out") .putOutputs("output", "go.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("go.out", pc("go.out")) .putTransforms( "pyTransform",
PTransform pt = PTransform.newBuilder() .putInputs("input", "input.out") .putInputs("side_input", "sideInput.in") .putInputs("timer", "timer.out") .putOutputs("output", "output.out") .putOutputs("timer", "timer.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build(); PCollection input = PCollection.newBuilder().setUniqueName("input.out").build(); PCollection sideInput = PCollection.newBuilder().setUniqueName("sideInput.in").build();
"read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "groupByKey", PTransform.newBuilder() .setUniqueName("GroupByKey") .putInputs("input", "read.out") .putOutputs("output", "groupByKey.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN)) .build()) .putPcollections("groupByKey.out", pc("groupByKey.out")) .putTransforms( "parDo", PTransform.newBuilder() .setUniqueName("ParDo") .putInputs("input", "groupByKey.out")