/** * Returns a {@link PBegin} owned by this Pipeline. This serves as the input of a root {@link * PTransform} such as {@link Read} or {@link Create}. */ public PBegin begin() { return PBegin.in(this); }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) { PBegin input = PBegin.in(transform.getPipeline()); BoundedSource<T> source; try { source = ReadTranslation.boundedSourceFromTransform(transform); } catch (IOException e) { throw new RuntimeException(e); } return PTransformReplacement.of(input, bounded(source)); }
@Test public void pushThenPopSucceeds() { TransformHierarchy.Node root = hierarchy.getCurrent(); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); assertThat(hierarchy.getCurrent(), equalTo(node)); hierarchy.popNode(); assertThat(node.finishedSpecifying, is(true)); assertThat(hierarchy.getCurrent(), equalTo(root)); }
final PCollection<Row> inputTable = PBegin.in(p).apply(Create.of(rows).withCoder(schema.getRowCoder()));
hierarchy.pushNode("Enclosing", PBegin.in(pipeline), enclosingPT); hierarchy.pushNode("Create", PBegin.in(pipeline), originalTransform); assertThat(hierarchy.getCurrent(), equalTo(original)); PCollection<Long> originalOutput = pipeline.apply(originalTransform); Read.Bounded<Long> replacementTransform = Read.from(CountingSource.upTo(1L)); PCollection<Long> replacementOutput = pipeline.apply(replacementTransform); Node replacement = hierarchy.replaceNode(original, PBegin.in(pipeline), replacementTransform); assertThat(hierarchy.getCurrent(), equalTo(replacement)); hierarchy.setOutput(replacementOutput);
/** * Builds an unbounded {@link PCollection} in {@link Pipeline} set by {@link * #inPipeline(Pipeline)}. * * <p>If timestamp field was set with {@link #withTimestampField(String)} then watermark will be * advanced to the values from that field. */ public PCollection<Row> buildUnbounded() { checkArgument(pipeline != null); checkArgument(rows.size() > 0); if (type == null) { type = rows.get(0).getSchema(); } TestStream.Builder<Row> values = TestStream.create( type, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rows) { if (timestampField != null) { values = values.advanceWatermarkTo(new Instant(row.getDateTime(timestampField))); } values = values.addElements(row); } return PBegin.in(pipeline).apply("unboundedPCollection", values.advanceWatermarkToInfinity()); } }
private PCollection<Row> prepareUnboundedPCollection2() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); Row row = rowsInTableA.get(0); values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); return PBegin.in(pipeline) .apply("unboundedInput2", values.advanceWatermarkToInfinity()) .apply( "unboundedInput2.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); } }
@Test public void producingOwnOutputWithCompositeFails() { final PCollection<Long> comp = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); PTransform<PBegin, PCollection<Long>> root = new PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return comp; } }; hierarchy.pushNode("Composite", PBegin.in(pipeline), root); Create.Values<Integer> create = Create.of(1); hierarchy.pushNode("Create", PBegin.in(pipeline), create); hierarchy.setOutput(pipeline.apply(create)); hierarchy.popNode(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("contains a primitive POutput produced by it"); thrown.expectMessage("primitive transforms are permitted to produce"); thrown.expectMessage("Composite"); hierarchy.setOutput(comp); }
@Test public void visitVisitsAllPushed() { TransformHierarchy.Node root = hierarchy.getCurrent(); PBegin begin = PBegin.in(pipeline);
@Test public void emptyCompositeSucceeds() { PCollection<Long> created = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode(); PCollectionList<Long> pcList = PCollectionList.of(created); TransformHierarchy.Node emptyTransform = hierarchy.pushNode( "Extract", pcList, new PTransform<PCollectionList<Long>, PCollection<Long>>() { @Override public PCollection<Long> expand(PCollectionList<Long> input) { return input.get(0); } }); hierarchy.setOutput(created); hierarchy.popNode(); assertThat(hierarchy.getProducer(created), equalTo(node)); assertThat( "A Transform that produces non-primitive output should be composite", emptyTransform.isCompositeNode(), is(true)); }
@Test public void testRegistrarEncodedProto() throws Exception { PCollection<String> output = p.apply(testStream); AppliedPTransform<PBegin, PCollection<String>, TestStream<String>> appliedTestStream = AppliedPTransform.of("fakeName", PBegin.in(p).expand(), output.expand(), testStream, p); SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); RunnerApi.FunctionSpec spec = PTransformTranslation.toProto(appliedTestStream, components).getSpec(); assertThat(spec.getUrn(), equalTo(TEST_STREAM_TRANSFORM_URN)); RunnerApi.TestStreamPayload payload = TestStreamPayload.parseFrom(spec.getPayload()); verifyTestStreamEncoding( testStream, payload, RehydratedComponents.forComponents(components.toComponents())); }
private PCollection<Row> prepareUnboundedPCollection1() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rowsInTableA) { values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); } return PBegin.in(pipeline) .apply("unboundedInput1", values.advanceWatermarkToInfinity()) .apply( "unboundedInput1.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); }
@Test public void testProject() { PCollection<PersonBean> input = PBegin.in(pipeline) .apply("input", Create.of(new PersonBean("Foo", 5), new PersonBean("Bar", 53))); String sql = "SELECT name FROM PCOLLECTION"; PCollection<Row> result = input.apply("sql", SqlTransform.query(sql)); PAssert.that(result) .containsInAnyOrder( TestUtils.rowsBuilderOf(Schema.builder().addStringField("name").build()) .addRows("Foo", "Bar") .getRows()); pipeline.run(); }
@Test public void testSelect() { PCollection<PersonBean> input = PBegin.in(pipeline) .apply("input", Create.of(new PersonBean("Foo", 5), new PersonBean("Bar", 53))); String sql = "SELECT name, ageYears FROM PCOLLECTION"; PCollection<Row> result = input.apply("sql", SqlTransform.query(sql)); PAssert.that(result) .containsInAnyOrder( TestUtils.rowsBuilderOf( Schema.builder().addStringField("name").addInt32Field("ageYears").build()) .addRows( "Foo", 5, "Bar", 53) .getRows()); pipeline.run(); }
PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode();
@Before public void setUp() { Schema schema = Schema.builder() .addNullableField("f_int1", Schema.FieldType.INT32) .addNullableField("f_int2", Schema.FieldType.INT32) .addInt32Field("f_int3") .build(); List<Row> rows = TestUtils.RowsBuilder.of(schema) .addRows(1, 5, 1) .addRows(null, 1, 1) .addRows(2, 1, 1) .addRows(null, 1, 1) .addRows(null, null, 1) .addRows(null, null, 1) .addRows(3, 2, 1) .getRows(); boundedInput = PBegin.in(pipeline).apply(Create.of(rows).withSchema(schema, identity(), identity())); }
PBegin.in(p) .apply( Create.of(row1, row2, row3)