@Override public PCollectionTuple expand(PBegin input) { return input.apply(producer); } });
@Override public PCollectionTuple expand(PBegin input) { return input.apply(producer); } });
@Override public PCollection<Row> buildIOReader(PBegin begin) { return begin .apply("ReadTextFiles", TextIO.read().from(filePattern)) .apply("StringToRow", readConverter); }
@Override public PCollection<T> expand(PBegin input) { getConfiguration().validate(); return input.apply(org.apache.beam.sdk.io.Read.from(createSource())); } }
@Override public PCollection<T> expand(PBegin input) { if (source.isBounded()) { org.apache.beam.sdk.io.Read.Bounded<T> bounded = org.apache.beam.sdk.io.Read.from(new BoundedSourceWrapper<>(source.asBounded(), this)); return input.apply(bounded); } else { throw new UnsupportedOperationException("Unbounded is not supported for now."); } }
@Override public PCollection<String> expand(PBegin begin) { return begin.apply(TextIO.read().from(inputFile)).apply(ParDo.of(new ExtractTimestamps())); } }
@Override public PCollection<Ddl> expand(PBegin p) { return p.apply("Create empty", Create.of((Void) null)) .apply( "Read Information Schema", ParDo.of(new ReadInformationSchemaFn(spannerConfig, tx)).withSideInputs(tx)); }
@Override public PDone expand(PBegin input) { return input .apply("Start signal", Create.of(START_SIGNAL_MESSAGE)) .apply(PubsubIO.writeStrings().to(startTopicPath.getPath())); } }
@Override public PCollection<String> expand(PBegin begin) { return begin.apply(TextIO.read().from(inputFile)).apply(ParDo.of(new ExtractTimestamps())); } }
@Override public PCollection<HCatRecord> expand(PBegin input) { checkArgument(getTable() != null, "withTable() is required"); checkArgument(getConfigProperties() != null, "withConfigProperties() is required"); return input.apply(org.apache.beam.sdk.io.Read.from(new BoundedHCatalogSource(this))); }
@Override public PCollection<MatchResult.Metadata> expand(PBegin input) { return input .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of())) .apply("Via MatchAll", matchAll().withConfiguration(getConfiguration())); } }
@Override public PCollection<ParseResult> expand(PBegin input) { return input .apply(FileIO.match().filepattern(getFilepattern())) .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED)) .apply(parseFiles()); } }
@Override public PCollectionView<Transaction> expand(PBegin input) { getSpannerConfig().validate(); return input .apply(Create.of(1)) .apply("Create transaction", ParDo.of(new CreateTransactionFn(this))) .apply("As PCollectionView", View.asSingleton()); }
@Override public PDone expand(PBegin begin) { return begin.apply(Create.of(LINES)).apply(TextIO.write().to(filename)); } }
@Override public PCollection<T> expand(PBegin input) { return input .apply(Create.of((Void) null)) .apply( ParDo.of( new DynamicReadFn<>( getDynamicDataSourceConfiguration(), getQuery(), getRowMapper()))) .setCoder(getCoder()) .apply(new Reparallelize<>()); }
@Override public PCollection<IndexedRecord> read(PBegin in) { LazyAvroCoder<IndexedRecord> lac = LazyAvroCoder.of(); ExcelHdfsFileSource source = ExcelHdfsFileSource.of(doAs, path, lac, limit, encoding, sheetName, header, footer, excelFormat.name()); source.getExtraHadoopConfiguration().addFrom(getExtraHadoopConfiguration()); source.setLimit(limit); PCollection<KV<Void, IndexedRecord>> pc1 = in.apply(Read.from(source)).setCoder(source.getDefaultOutputCoder()); PCollection<IndexedRecord> pc2 = pc1.apply(Values.<IndexedRecord>create()); return pc2; }
@Ignore @Test @Category(ValidatesRunner.class) public void testEmptyTransform() { p.begin().apply(new EmptyTransform()); p.run(); }
@Override public PCollection<SolrDocument> expand(PBegin input) { checkArgument( getConnectionConfiguration() != null, "withConnectionConfiguration() is required"); checkArgument(getCollection() != null, "from() is required"); return input .apply("Create", Create.of(this)) .apply("Split", ParDo.of(new SplitFn())) .apply("Reshuffle", Reshuffle.viaRandomKey()) .apply("Read", ParDo.of(new ReadFn())); }
@Override public PCollection<KV<String, String>> expand(PBegin input) { checkArgument(connectionConfiguration() != null, "withConnectionConfiguration() is required"); return input .apply(Create.of(keyPattern())) .apply(ParDo.of(new ReadKeysWithPattern(connectionConfiguration()))) .apply( RedisIO.readAll() .withConnectionConfiguration(connectionConfiguration()) .withBatchSize(batchSize())); } }
@Test public void testPartiallyBoundFailure() throws IOException { Pipeline p = Pipeline.create(buildPipelineOptions()); PCollection<Integer> input = p.begin().apply(Create.of(1, 2, 3)); thrown.expect(IllegalArgumentException.class); input.apply(new PartiallyBoundOutputCreator()); Assert.fail("Failure expected from use of partially bound output"); }