@Override public PCollection<TableRow> expand(PBegin input) { return input.apply(inner); }
@Override public PCollection<PubsubMessage> expand(PBegin input) { return input .getPipeline() .begin() .apply(Read.from(new PubsubSource(this))) .apply( "PubsubUnboundedSource.Stats", ParDo.of( new StatsFn(pubsubFactory, subscription, topic, timestampAttribute, idAttribute))); }
/** * Returns a {@link PBegin} owned by this Pipeline. This serves as the input of a root {@link * PTransform} such as {@link Read} or {@link Create}. */ public PBegin begin() { return PBegin.in(this); }
private PCollection<Row> prepareUnboundedPCollection2() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); Row row = rowsInTableA.get(0); values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); return PBegin.in(pipeline) .apply("unboundedInput2", values.advanceWatermarkToInfinity()) .apply( "unboundedInput2.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); } }
@Override public PCollection<T> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, coder); }
@Test public void testRegistrarEncodedProto() throws Exception { PCollection<String> output = p.apply(testStream); AppliedPTransform<PBegin, PCollection<String>, TestStream<String>> appliedTestStream = AppliedPTransform.of("fakeName", PBegin.in(p).expand(), output.expand(), testStream, p); SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); RunnerApi.FunctionSpec spec = PTransformTranslation.toProto(appliedTestStream, components).getSpec(); assertThat(spec.getUrn(), equalTo(TEST_STREAM_TRANSFORM_URN)); RunnerApi.TestStreamPayload payload = TestStreamPayload.parseFrom(spec.getPayload()); verifyTestStreamEncoding( testStream, payload, RehydratedComponents.forComponents(components.toComponents())); }
/** * Builds an unbounded {@link PCollection} in {@link Pipeline} set by {@link * #inPipeline(Pipeline)}. * * <p>If timestamp field was set with {@link #withTimestampField(String)} then watermark will be * advanced to the values from that field. */ public PCollection<Row> buildUnbounded() { checkArgument(pipeline != null); checkArgument(rows.size() > 0); if (type == null) { type = rows.get(0).getSchema(); } TestStream.Builder<Row> values = TestStream.create( type, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rows) { if (timestampField != null) { values = values.advanceWatermarkTo(new Instant(row.getDateTime(timestampField))); } values = values.addElements(row); } return PBegin.in(pipeline).apply("unboundedPCollection", values.advanceWatermarkToInfinity()); } }
@Override public PCollection<T> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, coder); }
@Override public PCollectionTuple expand(PBegin input) { return input.apply(producer); } });
@Override public PDone expand(PBegin input) { final PCollectionView<ActualT> actual = input.apply("CreateActual", createActual); input .apply(Create.of(0).withCoder(VarIntCoder.of())) .apply("WindowToken", windowToken) .apply( "RunChecks", ParDo.of(new SideInputCheckerDoFn<>(checkerFn, actual, site)).withSideInputs(actual)) .apply("VerifyAssertions", new DefaultConcludeTransform()); return PDone.in(input.getPipeline()); } }
private PCollection<Row> prepareUnboundedPCollection1() { TestStream.Builder<Row> values = TestStream.create( schemaInTableA, SerializableFunctions.identity(), SerializableFunctions.identity()); for (Row row : rowsInTableA) { values = values.advanceWatermarkTo(new Instant(row.getDateTime("f_timestamp"))); values = values.addElements(row); } return PBegin.in(pipeline) .apply("unboundedInput1", values.advanceWatermarkToInfinity()) .apply( "unboundedInput1.fixedWindow1year", Window.into(FixedWindows.of(Duration.standardDays(365)))); }
@Override public PCollection<T> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED, coder); } }
@Override public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) { PBegin input = PBegin.in(transform.getPipeline()); BoundedSource<T> source; try { source = ReadTranslation.boundedSourceFromTransform(transform); } catch (IOException e) { throw new RuntimeException(e); } return PTransformReplacement.of(input, bounded(source)); }
@Override public PCollectionTuple expand(PBegin input) { return input.apply(producer); } });
@Override public PCollectionList<String> expand(PBegin b) { // Composite transform: apply delegates to other transformations, // here a Create transform. PCollection<String> result = b.apply(Create.of("hello", "world")); // Issue below: PCollection.createPrimitiveOutput should not be used // from within a composite transform. return PCollectionList.of( Arrays.asList( result, PCollection.createPrimitiveOutputInternal( b.getPipeline(), WindowingStrategy.globalDefault(), result.isBounded(), StringUtf8Coder.of()))); } }
@Test public void testProject() { PCollection<PersonBean> input = PBegin.in(pipeline) .apply("input", Create.of(new PersonBean("Foo", 5), new PersonBean("Bar", 53))); String sql = "SELECT name FROM PCOLLECTION"; PCollection<Row> result = input.apply("sql", SqlTransform.query(sql)); PAssert.that(result) .containsInAnyOrder( TestUtils.rowsBuilderOf(Schema.builder().addStringField("name").build()) .addRows("Foo", "Bar") .getRows()); pipeline.run(); }
@Override public PCollection<byte[]> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, ByteArrayCoder.of()); } }
@Test public void pushThenPopSucceeds() { TransformHierarchy.Node root = hierarchy.getCurrent(); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); assertThat(hierarchy.getCurrent(), equalTo(node)); hierarchy.popNode(); assertThat(node.finishedSpecifying, is(true)); assertThat(hierarchy.getCurrent(), equalTo(root)); }
/** * Like {@link #apply(String, PTransform)} but the transform node in the {@link Pipeline} graph * will be named according to {@link PTransform#getName}. * * @see #apply(String, PTransform) */ public <OutputT extends POutput> OutputT apply(PTransform<? super PBegin, OutputT> root) { return begin().apply(root); }
@Override public PCollection<T> expand(PBegin input) { checkNotNull(getFilepattern(), "filepattern"); Coder<T> coder = inferCoder(getCoder(), getParseFn(), input.getPipeline().getCoderRegistry()); if (getMatchConfiguration().getWatchInterval() == null && !getHintMatchesManyFiles()) { return input.apply( org.apache.beam.sdk.io.Read.from( AvroSource.from(getFilepattern()).withParseFn(getParseFn(), coder))); } // All other cases go through ParseAllGenericRecords. return input .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of())) .apply( "Via ParseAll", parseAllGenericRecords(getParseFn()) .withCoder(coder) .withMatchConfiguration(getMatchConfiguration())); }