Components.Builder newComponents = Components.newBuilder(); String inputId = getOnlyElement(gbk.getInputsMap().values());
spk.getSpec().getUrn()); Components.Builder newComponents = Components.newBuilder(); newComponents.putAllCoders(components.getCodersMap());
.build(); RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut)
@Test public void flattenWithHeterogeneousInputsSingleEnvOutputPartiallyMaterialized() { Components components = Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build())
.build(); RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut)
.addRootTransformIds(flattenTransform.getUniqueName()) .setComponents( Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build())
.addAllRootTransformIds(ImmutableList.of("first", "second")) .setComponents( Components.newBuilder() .putTransforms( "first", WindowingStrategy.newBuilder().setAccumulationMode(Enum.ACCUMULATING).build(); RunnerApi.Components extraComponents = Components.newBuilder() .putPcollections( "intermediatePc",
Components.newBuilder() .putTransforms("pt", pt) .putPcollections("input.out", input)
.addRootTransformIds(flatten.getId()) .setComponents( RunnerApi.Components.newBuilder() .putTransforms(source.getId(), source.getTransform()) .putPcollections(left.getId(), left.getPCollection())
Components.newBuilder() .putTransforms( "impulse",
Components.newBuilder() .putTransforms("pt", pt) .putTransforms("other_pt", PTransform.newBuilder().setUniqueName("other").build())
.addAllRootTransformIds(ImmutableList.of("first", "second")) .setComponents( Components.newBuilder() .putTransforms( "first", .addSubtransforms(subtransform)) .setComponents( Components.newBuilder() .putTransforms( subtransform,
public void transformWithSameSideAndMainInput() { Components components = Components.newBuilder() .putPcollections("read_pc", RunnerApi.PCollection.getDefaultInstance()) .putPcollections("pardo_out", RunnerApi.PCollection.getDefaultInstance())
.addRootTransformIds("root") .setComponents( Components.newBuilder() .putTransforms( "root",
@Test public void forTransformsWithSubgraph() { Components components = Components.newBuilder() .putTransforms( "root", PTransform.newBuilder().putOutputs("output", "output.out").build()) .putPcollections( "output.out", RunnerApi.PCollection.newBuilder().setUniqueName("output.out").build()) .putTransforms( "consumer", PTransform.newBuilder().putInputs("input", "output.out").build()) .putTransforms( "ignored", PTransform.newBuilder().putInputs("input", "output.out").build()) .build(); QueryablePipeline pipeline = QueryablePipeline.forTransforms(ImmutableSet.of("root", "consumer"), components); assertThat( pipeline.getRootTransforms(), contains(PipelineNode.pTransform("root", components.getTransformsOrThrow("root")))); Set<PTransformNode> consumers = pipeline.getPerElementConsumers( PipelineNode.pCollection( "output.out", components.getPcollectionsOrThrow("output.out"))); assertThat( consumers, contains(PipelineNode.pTransform("consumer", components.getTransformsOrThrow("consumer")))); }
@Before public void setup() { partialComponents = Components.newBuilder() .putTransforms( "impulse", PTransform.newBuilder() .setUniqueName("Impulse") .putOutputs("output", "impulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("impulse.out", pc("impulse.out")) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .build(); }
@Test public void fromComponentsWithMalformedComponents() { Components components = Components.newBuilder() .putTransforms( "root", PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN) .build()) .putOutputs("output", "output.out") .build()) .build(); thrown.expect(IllegalArgumentException.class); QueryablePipeline.forPrimitivesIn(components).getComponents(); }
@Before public void setup() { partialComponents = Components.newBuilder() .putTransforms( "impulse", PTransform.newBuilder() .putOutputs("output", "impulse.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("impulse.out", impulseDotOut) .build(); }
@Test public void forTransformsWithMalformedGraph() { Components components = Components.newBuilder() .putTransforms( "root", PTransform.newBuilder().putOutputs("output", "output.out").build()) .putPcollections( "output.out", RunnerApi.PCollection.newBuilder().setUniqueName("output.out").build()) .putTransforms( "consumer", PTransform.newBuilder().putInputs("input", "output.out").build()) .build(); thrown.expect(IllegalArgumentException.class); // Consumer consumes a PCollection which isn't produced. QueryablePipeline.forTransforms(ImmutableSet.of("consumer"), components); }
private ExecutableGraphBuilder() { components = Components.newBuilder(); }