PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .putOutputs("out", redOut.getUniqueName()) .build(); PCollection threeOut = PCollection.newBuilder().setUniqueName("three.out").build(); PTransform three = PTransform.newBuilder() .putOutputs("out", threeOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder() .putOutputs("shared", sharedOut.getUniqueName())
input.toBuilder().setUniqueName(kwiCollectionId).setCoderId(kwiCoderId).build(); String rawGbkId = uniqueId(String.format("%s/RawGBK", spkId), components::containsTransforms); input.toBuilder().setUniqueName(feedSDFCollectionId).setCoderId(feedSDFCoderId).build(); String feedSDFId = uniqueId(String.format("%s/FeedSDF", spkId), components::containsTransforms);
"bounded", RunnerApi.PCollection.newBuilder() .setUniqueName("bounded") .setIsBounded(Enum.BOUNDED) .build())) .commit(Instant.now()), bundleFactory "unbounded", RunnerApi.PCollection.newBuilder() .setUniqueName("unbounded") .setIsBounded(Enum.UNBOUNDED) .build())) .commit(Instant.now())); CommittedResult<PTransformNode> result =
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder() .putOutputs("shared", sharedOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); PTransform blue = PTransform.newBuilder()
.build(); PCollection flattenPc = PCollection.newBuilder().setUniqueName("flatten.out").build(); Components components = partialComponents .putTransforms("pyRead", pyRead) .putPcollections( "pyRead.out", PCollection.newBuilder().setUniqueName("pyRead.out").build()) .putTransforms("goRead", goRead) .putPcollections( "goRead.out", PCollection.newBuilder().setUniqueName("goRead.out").build()) .putTransforms( "flatten", .putTransforms("pyParDo", pyParDo) .putPcollections( "pyParDo.out", PCollection.newBuilder().setUniqueName("pyParDo.out").build()) .putTransforms("goWindow", goWindow) .putPcollections( "goWindow.out", PCollection.newBuilder().setUniqueName("goWindow.out").build()) .putEnvironments("go", Environments.createDockerEnvironment("go")) .putEnvironments("py", Environments.createDockerEnvironment("py"))
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection sharedOut = PCollection.newBuilder().setUniqueName("shared.out").build(); PTransform shared = PTransform.newBuilder() .putOutputs("shared", sharedOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); PTransform blue = PTransform.newBuilder()
PCollection redOut = PCollection.newBuilder().setUniqueName("red.out").build(); PTransform red = PTransform.newBuilder() .putOutputs("out", redOut.getUniqueName()) .build(); PCollection oneOut = PCollection.newBuilder().setUniqueName("one.out").build(); PTransform one = PTransform.newBuilder() .putOutputs("out", oneOut.getUniqueName()) .build(); PCollection twoOut = PCollection.newBuilder().setUniqueName("two.out").build(); PTransform two = PTransform.newBuilder() .putOutputs("out", twoOut.getUniqueName()) .build(); PCollection blueOut = PCollection.newBuilder().setUniqueName("blue.out").build(); PTransform blue = PTransform.newBuilder()
.toBuilder() .putTransforms("read", readTransform) .putPcollections("read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms("envRead", otherEnvRead) .putPcollections( "envRead.out", PCollection.newBuilder().setUniqueName("envRead.out").build()) .putTransforms("flatten", flattenTransform) .putPcollections( "flatten.out", PCollection.newBuilder().setUniqueName("flatten.out").build()) .putTransforms("window", windowTransform) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .putEnvironments("rare", Environments.createDockerEnvironment("rare"))
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", .putPcollections( "side_read.out", PCollection.newBuilder().setUniqueName("side_read.out").build()) .putTransforms( "parDo", .build()) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms( "window", .build()) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", env) .build());
"values", RunnerApi.PCollection.newBuilder() .setUniqueName("values") .setCoderId(coderId) .setWindowingStrategyId(windowingStrategyId) .build()); PCollectionNode groupedKvs = PipelineNode.pCollection( "groupedKvs", RunnerApi.PCollection.newBuilder().setUniqueName("groupedKvs").build()); PTransformNode groupByKeyOnly = PipelineNode.pTransform(
.build(); PCollection sideInputPCollection = PCollection.newBuilder().setUniqueName("side_read.out").build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn( .putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "side_read", .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms("flatten", flattenTransform) .putPcollections( "flatten.out", PCollection.newBuilder().setUniqueName("flatten.out").build()) .putTransforms("window", windowTransform) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("common", Environments.createDockerEnvironment("common")) .build());
.putPcollections( "intermediatePc", PCollection.newBuilder().setUniqueName("intermediate").build()) .putCoders( "coder", .putPcollections( "intermediatePc", PCollection.newBuilder().setUniqueName("intermediate_replacement").build()) .putWindowingStrategies("new_ws", introducedWS) .putTransforms("second_sub", PTransform.getDefaultInstance())
.toByteString())) .build(); PCollection input = PCollection.newBuilder().setUniqueName("input.out").build(); PCollection sideInput = PCollection.newBuilder().setUniqueName("sideInput.in").build(); PCollection timer = PCollection.newBuilder().setUniqueName("timer.out").build(); PCollection output = PCollection.newBuilder().setUniqueName("output.out").build();
.toByteString())) .build(); PCollection input = PCollection.newBuilder().setUniqueName("input.out").build(); PCollection sideInput = PCollection.newBuilder().setUniqueName("sideInput.in").build(); PCollection timer = PCollection.newBuilder().setUniqueName("timer.pc").build(); PCollection output = PCollection.newBuilder().setUniqueName("output.out").build();
.putPcollections( "intermediatePc", PCollection.newBuilder().setUniqueName("intermediate").build()) .putCoders( "coder",
.build(); PCollection sidePC = PCollection.newBuilder().setUniqueName("sidePC").build(); PCollection impulsePC = PCollection.newBuilder().setUniqueName("impulsePC").build(); QueryablePipeline p = QueryablePipeline.forPrimitivesIn(
.build(); PCollection userStateMainInputPCollection = PCollection.newBuilder().setUniqueName("read.out").build(); .putPcollections( "user_state.out", PCollection.newBuilder().setUniqueName("user_state.out").build()) .putTransforms("parDo", parDoTransform) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putEnvironments("common", env) .build());
@Test public void testFlattenInMemoryEvaluator() throws Exception { PCollectionNode left = PipelineNode.pCollection("left", PCollection.newBuilder().setUniqueName("left").build()); PCollectionNode right = PipelineNode.pCollection("right", PCollection.newBuilder().setUniqueName("right").build()); PipelineNode.pCollection("flat", PCollection.newBuilder().setUniqueName("flat").build()); PTransformNode flatten = PipelineNode.pTransform(
.putTransforms("read", readTransform) .putPcollections( "read.out", PCollection.newBuilder().setUniqueName("read.out").build()) .putTransforms( "parDo", .build()) .putPcollections( "parDo.out", PCollection.newBuilder().setUniqueName("parDo.out").build()) .putTransforms( "window", .build()) .putPcollections( "window.out", PCollection.newBuilder().setUniqueName("window.out").build()) .putEnvironments("rare", Environments.createDockerEnvironment("rare")) .putEnvironments("common", env)