newComponents.putAllCoders(components.getCodersMap()); .putCoders(kwiCoderId, kwiCoder) .putPcollections(kwiCollectionId, kwiCollection) .putTransforms(rawGbkId, rawGbk); newPTransform.addSubtransforms(rawGbkId); .putPcollections(feedSDFCollectionId, feedSDFCollection) .putTransforms(feedSDFId, feedSDF); newPTransform.addSubtransforms(feedSDFId); .putAllOutputs(spk.getOutputsMap()) .build(); newComponents.putTransforms(runSDFId, runSDF); newPTransform.addSubtransforms(runSDFId);
RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut) .putTransforms("red", red) .putPcollections(redOut.getUniqueName(), redOut) .putTransforms("shared", shared) .putPcollections(sharedOut.getUniqueName(), sharedOut) .putTransforms("blue", blue) .putPcollections(blueOut.getUniqueName(), blueOut) .build(); PTransformNode sharedTransform = PipelineNode.pTransform("shared", shared); ExecutableStage oneStage =
/** * Registers the provided {@link WindowingStrategy} into this {@link SdkComponents}, returning a * unique ID for the {@link WindowingStrategy}. Multiple registrations of the same {@link * WindowingStrategy} will return the same unique ID. */ public String registerWindowingStrategy(WindowingStrategy<?, ?> windowingStrategy) throws IOException { String existing = windowingStrategyIds.get(windowingStrategy); if (existing != null) { return existing; } String baseName = String.format( "%s(%s)", NameUtils.approximateSimpleName(windowingStrategy), NameUtils.approximateSimpleName(windowingStrategy.getWindowFn())); String name = uniqify(baseName, windowingStrategyIds.values()); windowingStrategyIds.put(windowingStrategy, name); RunnerApi.WindowingStrategy windowingStrategyProto = WindowingStrategyTranslation.toProto(windowingStrategy, this); componentsBuilder.putWindowingStrategies(name, windowingStrategyProto); return name; }
RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut) .putTransforms("two", two) .putPcollections(twoOut.getUniqueName(), twoOut) .putTransforms("multi", three) .putPcollections(threeOut.getUniqueName(), threeOut) .putTransforms("shared", shared) .putPcollections(sharedOut.getUniqueName(), sharedOut) .putTransforms("otherShared", otherShared) .putPcollections(otherSharedOut.getUniqueName(), otherSharedOut) .putTransforms("red", red) .putPcollections(redOut.getUniqueName(), redOut) .putTransforms("blue", blue) .putPcollections(blueOut.getUniqueName(), blueOut) .build(); ExecutableStage multiStage = ImmutableExecutableStage.of(
Components components = Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .putTransforms( "pyImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .putTransforms( "pyRead", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .putTransforms( "goImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("goImpulse.out", pc("goImpulse.out")) .putTransforms( "goRead",
RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut) .putTransforms("two", two) .putPcollections(twoOut.getUniqueName(), twoOut) .putTransforms("shared", shared) .putPcollections(sharedOut.getUniqueName(), sharedOut) .putTransforms("red", red) .putPcollections(redOut.getUniqueName(), redOut) .putTransforms("blue", blue) .putPcollections(blueOut.getUniqueName(), blueOut) .build(); ExecutableStage oneStage = ImmutableExecutableStage.of(
Components components = Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .putTransforms( "mainImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("mainImpulse.out", pc("mainImpulse.out")) .putTransforms( "read", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "sideImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("sideImpulse.out", pc("sideImpulse.out")) .putTransforms( "sideRead",
partialComponents .toBuilder() .putTransforms("pyRead", pyRead) .putPcollections( "pyRead.out", PCollection.newBuilder().setUniqueName("pyRead.out").build()) .putTransforms("goRead", goRead) .putPcollections( "goRead.out", PCollection.newBuilder().setUniqueName("goRead.out").build()) .putTransforms( "flatten", PTransform.newBuilder() .build()) .build()) .putPcollections("flatten.out", flattenPc) .putTransforms("pyParDo", pyParDo) .putPcollections( "pyParDo.out", PCollection.newBuilder().setUniqueName("pyParDo.out").build()) .putTransforms("goWindow", goWindow) .putPcollections( "goWindow.out", PCollection.newBuilder().setUniqueName("goWindow.out").build()) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py")) .build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn(components);
Components components = Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .putTransforms( "pyImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .putTransforms( "pyRead", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("pyRead.out", pc("pyRead.out")) .putTransforms( "goImpulse", PTransform.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("goImpulse.out", pc("goImpulse.out")) .putTransforms( "goRead",
.clearTransforms() .putAllTransforms( pTransformNodes .stream() componentBuilder.clearPcollections().putAllPcollections(validPCollectionMap); componentBuilder.build(), stage.getEnvironment(), stage.getInputPCollection(),
partialComponents .toBuilder() .putTransforms("read", readTransform) .putPcollections("read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms("envRead", otherEnvRead) .putPcollections( "envRead.out", PCollection.newBuilder().setUniqueName("envRead.out").build()) .putTransforms("flatten", flattenTransform) .putPcollections( "flatten.out", PCollection.newBuilder().setUniqueName("flatten.out").build()) .putTransforms("window", windowTransform) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .putEnvironments("rare", Environments.createDockerEnvironment("rare")) .build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn(components);
RunnerApi.Components components = Components.newBuilder() .putTransforms("one", one) .putPcollections(oneOut.getUniqueName(), oneOut) .putTransforms("two", two) .putPcollections(twoOut.getUniqueName(), twoOut) .putTransforms("red", red) .putPcollections(redOut.getUniqueName(), redOut) .putTransforms("blue", blue) .putPcollections(blueOut.getUniqueName(), blueOut) .build(); ExecutableStage oneStage = ImmutableExecutableStage.of(
partialComponents .toBuilder() .putTransforms( "read", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "goTransform", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("go.out", pc("go.out")) .putTransforms( "pyTransform", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("py.out", pc("py.out")) .putTransforms( "compositeMultiLang", PTransform.newBuilder() .addSubtransforms("pyTransform") .build())
originalToPartial.put( originalToPartialReplacement.getKey(), originalToPartialReplacement.getValue()); unzippedComponents.putPcollections( originalToPartialReplacement.getValue().getId(), originalToPartialReplacement.getValue().getPCollection()); originalToPartial.put( originalToPartialReplacement.getKey(), originalToPartialReplacement.getValue()); unzippedComponents.putPcollections( originalToPartialReplacement.getValue().getId(), originalToPartialReplacement.getValue().getPCollection()); createFlattenOfPartials( flattenId, partialFlattenTargets.getKey(), partialFlattenTargets.getValue()); unzippedComponents.putTransforms(flattenId, flattenPartialPCollections); introducedFlattens.add(PipelineNode.pTransform(flattenId, flattenPartialPCollections)); Components components = unzippedComponents.build(); return DeduplicationResult.of(components, introducedFlattens, updatedStages, updatedTransforms);
String runnerWireCoderId = LengthPrefixUnknownCoders.addLengthPrefixedCoder(coderId, builder, true); RehydratedComponents components = RehydratedComponents.forComponents(builder.build()); Coder<KV<String, Integer>> javaWireCoder = (Coder<KV<String, Integer>>) components.getCoder(javaWireCoderId); .setComponents( builder .putTransforms(inputTransform.getId(), inputTransform.getTransform()) .putTransforms(groupByKeyOnly.getId(), groupByKeyOnly.getTransform()) .putPcollections(values.getId(), values.getPCollection()) .putPcollections(groupedKvs.getId(), groupedKvs.getPCollection())) .build();
.setComponents( Components.newBuilder() .putCoders("coder", Coder.newBuilder().build()) .putCoders("windowCoder", Coder.newBuilder().build()) .putWindowingStrategies( "ws", WindowingStrategy.newBuilder().setWindowCoderId("windowCoder").build()) .putEnvironments("py", Environments.createDockerEnvironment("py")) .putPcollections(flattenOutput.getUniqueName(), flattenOutput) .putTransforms(flattenTransform.getUniqueName(), flattenTransform) .putPcollections(read1Output.getUniqueName(), read1Output) .putTransforms(read1Transform.getUniqueName(), read1Transform) .putPcollections(read2Output.getUniqueName(), read2Output) .putTransforms(read2Transform.getUniqueName(), read2Transform) .putPcollections(impulse1Output.getUniqueName(), impulse1Output) .putTransforms(impulse1Transform.getUniqueName(), impulse1Transform) .putPcollections(impulse2Output.getUniqueName(), impulse2Output) .putTransforms(impulse2Transform.getUniqueName(), impulse2Transform) .build()) .build(); FusedPipeline fused = GreedyPipelineFuser.fuse(impulse);
.setComponents( RunnerApi.Components.newBuilder() .putTransforms(source.getId(), source.getTransform()) .putPcollections(left.getId(), left.getPCollection()) .putPcollections(right.getId(), right.getPCollection()) .putTransforms(flatten.getId(), flatten.getTransform()) .putPcollections(flattened.getId(), flattened.getPCollection())) .build());
partialComponents .toBuilder() .putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", PTransform.newBuilder() .putOutputs("output", "side_read.out") .build()) .putPcollections("side_read.out", sideInputPCollection) .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
partialComponents .toBuilder() .putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", PTransform.newBuilder() .putOutputs("output", "side_read.out") .build()) .putPcollections( "side_read.out", PCollection.newBuilder().setUniqueName("side_read.out").build()) .putTransforms( "parDo", PTransform.newBuilder() .toByteString())) .build()) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms( "window", PTransform.newBuilder() .toByteString())) .build()) .putPcollections(
partialComponents .toBuilder() .putTransforms( "read", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("read.out", pc("read.out")) .putTransforms( "goTransform", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("go.out", pc("go.out")) .putTransforms( "pyTransform", PTransform.newBuilder() .toByteString())) .build()) .putPcollections("py.out", pc("py.out")) .build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());