transformBuilder.setUniqueName(appliedPTransform.getFullName()); transformBuilder.setDisplayData( DisplayDataTranslation.toProto(DisplayData.from(appliedPTransform.getTransform())));
private static PTransform createFlattenOfPartials( String transformId, String outputId, Collection<PCollectionNode> generatedInputs) { PTransform.Builder newFlattenBuilder = PTransform.newBuilder(); int i = 0; for (PCollectionNode generatedInput : generatedInputs) { String localInputId = String.format("input_%s", i); i++; newFlattenBuilder.putInputs(localInputId, generatedInput.getId()); } // Flatten all of the new partial nodes together. return newFlattenBuilder // Use transform ID as unique name. .setUniqueName(transformId) .putOutputs("output", outputId) .setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN)) .build(); }
PTransform gbko = PTransform.newBuilder() .setUniqueName(String.format("%s/GBKO", gbk.getUniqueName())) .putAllInputs(gbk.getInputsMap()) .setSpec(FunctionSpec.newBuilder().setUrn(DirectGroupByKey.DIRECT_GBKO_URN)) PTransform gabw = PTransform.newBuilder() .setUniqueName(String.format("%s/GABW", gbk.getUniqueName())) .putInputs("input", kwiCollectionId) .setSpec(FunctionSpec.newBuilder().setUrn(DirectGroupByKey.DIRECT_GABW_URN))
PTransform readTransform = PTransform.newBuilder() .setUniqueName("read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") .setUniqueName("parDo") .putInputs("input", "read.out") .putInputs("side_input", "side_read.out") "side_read", PTransform.newBuilder() .setUniqueName("side_read") .setSpec( FunctionSpec.newBuilder()
"read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") "groupByKey", PTransform.newBuilder() .setUniqueName("GroupByKey") .putInputs("input", "read.out") .putOutputs("output", "groupByKey.out") "parDo", PTransform.newBuilder() .setUniqueName("ParDo") .putInputs("input", "groupByKey.out") .putOutputs("output", "parDo.out")
"flatten", PTransform.newBuilder() .setUniqueName("Flatten") .putInputs("impulseInput", "impulse.out") .putOutputs("output", "flatten.out") "read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "flatten.out") .putOutputs("output", "read.out") "parDo", PTransform.newBuilder() .setUniqueName("ParDo") .putInputs("input", "read.out") .putOutputs("output", "parDo.out")
"flatten", PTransform.newBuilder() .setUniqueName("flatten") .putInputs("left", left.getId()) .putInputs("right", right.getId())
PTransform impulse = PTransform.newBuilder() .setUniqueName("impulse") .putOutputs("output", "impulsePC") .setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) PTransform createSide = PTransform.newBuilder() .setUniqueName("createSide") .putInputs("input", "impulsePC") .putOutputs("output", "sidePC") PTransform processMain = PTransform.newBuilder() .setUniqueName("processMain") .putInputs("main", "impulsePC") .putInputs("side", "sidePC")
"read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") "parDo", PTransform.newBuilder() .setUniqueName("ParDo") .putInputs("input", "read.out") .putOutputs("output", "parDo.out") "window", PTransform.newBuilder() .setUniqueName("Window") .putInputs("input", "parDo.out") .putOutputs("output", "window.out")
.setUniqueName("ParDo") .putInputs("input", "impulse.out") .putOutputs("output", "parDo.out") PTransform statefulTransform = PTransform.newBuilder() .setUniqueName("StatefulParDo") .putInputs("input", "parDo.out") .putOutputs("output", "stateful.out")
PTransform.Builder pt = PTransform.newBuilder().setUniqueName(uniqueName); ExecutableStagePayload.Builder payload = ExecutableStagePayload.newBuilder();
"read", PTransform.newBuilder() .setUniqueName("Read") .putInputs("input", "impulse.out") .putOutputs("output", "read.out") "goTransform", PTransform.newBuilder() .setUniqueName("GoTransform") .putInputs("input", "read.out") .putOutputs("output", "go.out") "pyTransform", PTransform.newBuilder() .setUniqueName("PyTransform") .putInputs("input", "read.out") .putOutputs("output", "py.out")
.setUniqueName("ParDo") .putInputs("input", "impulse.out") .putOutputs("output", "parDo.out") PTransform timerTransform = PTransform.newBuilder() .setUniqueName("TimerParDo") .putInputs("input", "parDo.out") .putInputs("timer", "timer.out")
Components.newBuilder() .putTransforms("pt", pt) .putTransforms("other_pt", PTransform.newBuilder().setUniqueName("other").build()) .putPcollections("input.out", input) .putPcollections("sideInput.in", sideInput)
.putTransforms( subtransform, PTransform.newBuilder().setUniqueName(subtransform).build())) .build(); });
PTransform timerTransform = PTransform.newBuilder() .setUniqueName("TimerParDo") .putInputs("input", "impulse.out") .putInputs("timer", "timer.out")
FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)) .setUniqueName("Mystery") .putInputs("input", "impulse.out") .putOutputs("output", "mystery.out") FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)) .setUniqueName("Enigma") .putInputs("input", "impulse.out") .putOutputs("output", "enigma.out")
@Before public void setup() { partialComponents = Components.newBuilder() .putTransforms( "impulse", PTransform.newBuilder() .setUniqueName("Impulse") .putOutputs("output", "impulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("impulse.out", pc("impulse.out")) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .build(); }
@Before public void setup() { create = PipelineNode.pTransform( "create", PTransform.newBuilder() .setUniqueName("create") .putInputs("in", "impulse.out") .putOutputs("out", "create.out") .build()); sum = PipelineNode.pTransform( "sum", PTransform.newBuilder() .setUniqueName("sum") .putInputs("in", "create.in") .putOutputs("out", "sum.out") .build()); }
public ExecutableGraphBuilder addTransform( String name, @Nullable String input, String... outputs) { PTransform.Builder pt = PTransform.newBuilder().setUniqueName(name); if (input != null) { pt = pt.putInputs("input", input); addPCollection(input); } for (String output : outputs) { pt = pt.putOutputs(output, output); addPCollection(output); } components.putTransforms(name, pt.build()); return this; }