/** * Return a map of IDs to {@link PTransform} which are executed by an SDK Harness. * * <p>The transforms that are present in the returned map are the {@link RunnerApi.PTransform} * versions of the {@link ExecutableStage ExecutableStages} returned in {@link #getFusedStages()}. * The IDs of the returned transforms will not collide with any transform ID present in {@link * #getComponents()}. */ private Map<String, PTransform> getEnvironmentExecutedTransforms() { Map<String, PTransform> topLevelTransforms = new HashMap<>(); for (ExecutableStage stage : getFusedStages()) { String baseName = String.format( "%s/%s", stage.getInputPCollection().getPCollection().getUniqueName(), stage.getEnvironment().getUrn()); Set<String> usedNames = Sets.union(topLevelTransforms.keySet(), getComponents().getTransformsMap().keySet()); String uniqueId = SyntheticComponents.uniqueId(baseName, usedNames::contains); topLevelTransforms.put(uniqueId, stage.toPTransform(uniqueId)); } return topLevelTransforms; } }
PCollection pc = components.getPcollectionsOrThrow(pcollectionId); checkArgument( !pc.getUniqueName().isEmpty(), "%s: PCollection %s does not have a unique_name set", context, pcollectionId); String previousId = uniqueNamesById.put(pc.getUniqueName(), pcollectionId); checkArgument( previousId == null, pcollectionId, previousId, pc.getUniqueName()); checkArgument( components.containsCoders(pc.getCoderId()),
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("red", redOut.getUniqueName()) .putOutputs("shared", sharedOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", sharedOut.getUniqueName()) .putOutputs("out", blueOut.getUniqueName()) .build(); RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one)
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection threeOut = PCollection.newBuilder().setUniqueName("three.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", threeOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build())
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("two", twoOut.getUniqueName()) .putOutputs("shared", sharedOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)
.setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("two", twoOut.getUniqueName()) .putOutputs("out", blueOut.getUniqueName()) .build(); RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one)
PTransform.newBuilder() .setUniqueName("Flatten") .putInputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .putInputs(read2Output.getUniqueName(), read2Output.getUniqueName()) .putOutputs(flattenOutput.getUniqueName(), flattenOutput.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("read1") .putInputs(impulse1Output.getUniqueName(), impulse1Output.getUniqueName()) .putOutputs(read1Output.getUniqueName(), read1Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("read2") .putInputs(impulse2Output.getUniqueName(), impulse2Output.getUniqueName()) .putOutputs(read2Output.getUniqueName(), read2Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder() .setUniqueName("impulse1") .putOutputs(impulse1Output.getUniqueName(), impulse1Output.getUniqueName()) .setSpec( FunctionSpec.newBuilder() PTransform.newBuilder()
updated.getComponents().getPcollectionsOrThrow("intermediatePc").getUniqueName(), equalTo("intermediate_replacement"));