newPipeline.addAllRootTransformIds( pipeline .getRootTransformIdsList() .stream() .filter(id -> !viewTransforms.contains(id))
feedSDFNodes.stream().map(PTransformNode::getId).collect(Collectors.toSet()); newPipeline.clearRootTransformIds(); for (String rootId : p.getRootTransformIdsList()) { if (!feedSDFIds.contains(rootId)) { newPipeline.addRootTransformIds(rootId);
@Override public void translate(StreamingTranslationContext context, RunnerApi.Pipeline pipeline) { QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault(transform.getTransform().getSpec().getUrn(), this::urnNotFound) .translate(transform.getId(), pipeline, context); } }
updated.getComponents().getCodersOrThrow("coder"), equalTo(p.getComponents().getCodersOrThrow("coder"))); assertThat(updated.getRootTransformIdsList(), equalTo(p.getRootTransformIdsList()));
checkState( protoPipeline .getRootTransformIdsList() .containsAll(ImmutableList.of("impulse", "map", "key", "gbk", "values")), "Unexpected Root Transform IDs %s", protoPipeline.getRootTransformIdsList()); "Root Transforms should all be present in the Pipeline Components", fusedPipelineProto.getComponents().getTransformsMap().keySet(), hasItems(fusedPipelineProto.getRootTransformIdsList().toArray(new String[0]))); assertThat( "Should contain Impulse, GroupByKey, and two Environment Stages", fusedPipelineProto.getRootTransformIdsCount(), equalTo(4)); assertThat(fusedPipelineProto.getRootTransformIdsList(), hasItems("impulse", "gbk")); assertRootsInTopologicalOrder(fusedPipelineProto); assertThat( "Fused transforms shouldn't be present in the root IDs", fusedPipelineProto.getRootTransformIdsList(), not(hasItems(startsWith("map"), startsWith("key"), startsWith("values"))));
PTransformNode stage = fusedPipeline .getRootTransformIdsList() .stream() .map(
PTransformNode stage = fusedPipeline .getRootTransformIdsList() .stream() .map(
@Override public FlinkPortablePipelineTranslator.Executor translate( BatchTranslationContext context, RunnerApi.Pipeline pipeline) { // Use a QueryablePipeline to traverse transforms topologically. QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault( transform.getTransform().getSpec().getUrn(), FlinkBatchPortablePipelineTranslator::urnNotFound) .translate(transform, pipeline, context); } // Ensure that side effects are performed for unconsumed DataSets. for (DataSet<?> dataSet : context.getDanglingDataSets()) { dataSet.output(new DiscardingOutputFormat<>()).name("DiscardingOutput"); } return context; }
@Override public void translate(BatchTranslationContext context, RunnerApi.Pipeline pipeline) { // Use a QueryablePipeline to traverse transforms topologically. QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault( transform.getTransform().getSpec().getUrn(), FlinkBatchPortablePipelineTranslator::urnNotFound) .translate(transform, pipeline, context); } // Ensure that side effects are performed for unconsumed DataSets. for (DataSet<?> dataSet : context.getDanglingDataSets()) { dataSet.output(new DiscardingOutputFormat<>()); } }
private PortableGraph(RunnerApi.Pipeline p) { this.queryablePipeline = QueryablePipeline.forTransforms(p.getRootTransformIdsList(), p.getComponents()); }
@Override public FlinkPortablePipelineTranslator.Executor translate( StreamingTranslationContext context, RunnerApi.Pipeline pipeline) { QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault(transform.getTransform().getSpec().getUrn(), this::urnNotFound) .translate(transform.getId(), pipeline, context); } return context; }
public static void validate(RunnerApi.Pipeline p) { Components components = p.getComponents(); for (String transformId : p.getRootTransformIdsList()) { checkArgument( components.containsTransforms(transformId), "Root transform id %s is unknown", transformId); } validateComponents("pipeline", components); }
/** * Create a new {@link QueryablePipeline} which uses the root transform IDs and components of the * provided {@link Pipeline}. */ public static QueryablePipeline forPipeline(RunnerApi.Pipeline p) { return forTransforms(p.getRootTransformIdsList(), p.getComponents()); }