.setComponents(fusedComponents) .addAllRootTransformIds(rootTransformIds) .build();
/** * Update all composites present in the {@code originalPipeline} with an URN equal to the provided * {@code urn} using the provided {@link TransformReplacement}. */ public static Pipeline updateTransform( String urn, Pipeline originalPipeline, TransformReplacement compositeBuilder) { Components.Builder resultComponents = originalPipeline.getComponents().toBuilder(); for (Map.Entry<String, PTransform> pt : originalPipeline.getComponents().getTransformsMap().entrySet()) { if (pt.getValue().getSpec() != null && urn.equals(pt.getValue().getSpec().getUrn())) { MessageWithComponents updated = compositeBuilder.getReplacement(pt.getKey(), originalPipeline.getComponents()); checkArgument( updated.getPtransform().getOutputsMap().equals(pt.getValue().getOutputsMap()), "A %s must produce all of the outputs of the original %s", TransformReplacement.class.getSimpleName(), PTransform.class.getSimpleName()); removeSubtransforms(pt.getValue(), resultComponents); resultComponents .mergeFrom(updated.getComponents()) .putTransforms(pt.getKey(), updated.getPtransform()); } } return originalPipeline.toBuilder().setComponents(resultComponents).build(); }
.filter(id -> !viewTransforms.contains(id)) .collect(Collectors.toList())); return newPipeline.build();
.putPcollections(values.getId(), values.getPCollection()) .putPcollections(groupedKvs.getId(), groupedKvs.getPCollection())) .build();
"coder", Coder.newBuilder().setSpec(SdkFunctionSpec.getDefaultInstance()).build())) .build();
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.putTransforms(flatten.getId(), flatten.getTransform()) .putPcollections(flattened.getId(), flattened.getPCollection())) .build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
"coder", Coder.newBuilder().setSpec(SdkFunctionSpec.getDefaultInstance()).build())) .build();
GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
return newPipeline.build();
.setComponents(components.toComponents()) .addAllRootTransformIds(rootIds) .build(); if (!useDeprecatedViewTransforms) {
.setSpec(FunctionSpec.newBuilder().setUrn("beam:inner")) .build())) .build();
GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
private RunnerApi.Pipeline makeKnownUrnsPrimitives( RunnerApi.Pipeline pipeline, Set<String> knownUrns) { RunnerApi.Pipeline.Builder trimmedPipeline = pipeline.toBuilder(); for (String ptransformId : pipeline.getComponents().getTransformsMap().keySet()) { if (knownUrns.contains( pipeline.getComponents().getTransformsOrThrow(ptransformId).getSpec().getUrn())) { LOG.debug("Removing descendants of known PTransform {}" + ptransformId); removeDescendants(trimmedPipeline, ptransformId); } } return trimmedPipeline.build(); }
public ExecutableGraph<PTransformNode, PCollectionNode> toGraph() { return PortableGraph.forPipeline( Pipeline.newBuilder() .setComponents(components) .addAllRootTransformIds(components.getTransformsMap().keySet()) .build()); } }