private static MapElements<Row, Void> logRecords(String suffix) { return MapElements.via( new SimpleFunction<Row, Void>() { @Override public @Nullable Void apply(Row input) { System.out.println(input.getValues() + suffix); return null; } }); }
@Override public PCollection<String> expand(PCollection<? extends Iterable<?>> input) { return input.apply( MapElements.via( new SimpleFunction<Iterable<?>, String>() { @Override public String apply(Iterable<?> input) { return Joiner.on(delimiter).join(input); } })); } }
@Override public PCollection<String> expand(PCollection<?> input) { return input.apply( MapElements.via( new SimpleFunction<Object, String>() { @Override public String apply(Object input) { return input.toString(); } })); } }
@Override public PCollection<String> expand(PCollection<? extends KV<?, ?>> input) { return input.apply( MapElements.via( new SimpleFunction<KV<?, ?>, String>() { @Override public String apply(KV<?, ?> input) { return input.getKey().toString() + delimiter + input.getValue().toString(); } })); } }
private static PTransform<PCollection<? extends String>, PCollection<String>> addSuffix( final String suffix) { return MapElements.via( new SimpleFunction<String, String>() { @Override public String apply(String input) { return input + suffix; } }); }
.apply(MapElements.via(new SimpleFunction<String, MutationGroup>() {
@SuppressWarnings("UnusedReturnValue") private static PCollection<String> addTransform(final PCollection<String> pCollection) { return pCollection.apply( "Map2", MapElements.via( new SimpleFunction<String, String>() { @Override public String apply(final String input) { return WHATEVER; } })); }
@Override public PCollection<KV<K, Void>> expand(PCollection<KV<K, V>> input) { return input.apply( MapElements.via( new SimpleFunction<KV<K, V>, KV<K, Void>>() { @Override public KV<K, Void> apply(KV<K, V> input) { return KV.of(input.getKey(), null); } })); } }
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); SpannerConfig spannerConfig = SpannerConfig.create() .withInstanceId(options.getInstanceId()) .withDatabaseId(options.getDatabaseId()); // [START spanner_dataflow_readall] PCollection<Struct> allRecords = p.apply(SpannerIO.read() .withSpannerConfig(spannerConfig) .withQuery("SELECT t.table_name FROM information_schema.tables AS t WHERE t" + ".table_catalog = '' AND t.table_schema = ''")).apply( MapElements.into(TypeDescriptor.of(ReadOperation.class)) .via((SerializableFunction<Struct, ReadOperation>) input -> { String tableName = input.getString(0); return ReadOperation.create().withQuery("SELECT * FROM " + tableName); })).apply(SpannerIO.readAll().withSpannerConfig(spannerConfig)); // [END spanner_dataflow_readall] PCollection<Long> dbEstimatedSize = allRecords.apply(EstimateSize.create()) .apply(Sum.longsGlobally()); dbEstimatedSize.apply(ToString.elements()).apply(TextIO.write().to(options.getOutput()) .withoutSharding()); p.run().waitUntilFinish(); }
@Override public PCollection<String> expand(PCollection<Row> input) { return input.apply( "rowToCsv", MapElements.into(TypeDescriptors.strings()).via(row -> beamRow2CsvLine(row, csvFormat))); } }
@Override public PCollection<T> expand(PCollection<T> input) { List<PCollectionView<?>> views = Lists.newArrayList(); for (int i = 0; i < signals.size(); ++i) { views.add(signals.get(i).apply("To wait view " + i, new ToWaitView())); } return input.apply( "Wait", MapElements.into(input.getCoder().getEncodedTypeDescriptor()) .via(fn((t, c) -> t, requiresSideInputs(views)))); } }
@Override public PCollection<StreamReadsRequest> expand(PCollection<String> lines) { return lines.apply(ParDo.of(new SitesToContigsFn())) .apply("Contigs to StreamReadsRequests", MapElements.via(new ContigsToStreamReadsRequestsFn(prototype))); } }
@Test public void testSimpleFunctionClassDisplayData() { SimpleFunction<?, ?> simpleFn = new SimpleFunction<Integer, Integer>() { @Override public Integer apply(Integer input) { return input; } }; MapElements<?, ?> simpleMap = MapElements.via(simpleFn); assertThat(DisplayData.from(simpleMap), hasDisplayItem("class", simpleFn.getClass())); }
@Test public void testSerializableFunctionDisplayData() { SerializableFunction<Integer, Integer> serializableFn = input -> input; MapElements<?, ?> serializableMap = MapElements.into(integers()).via(serializableFn); assertThat( DisplayData.from(serializableMap), hasDisplayItem("class", serializableFn.getClass())); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> gameInfo) { return gameInfo .apply( MapElements.into( TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.integers())) .via((GameActionInfo gInfo) -> KV.of(gInfo.getKey(field), gInfo.getScore()))) .apply(Sum.integersPerKey()); } }
@Test public void testMapOutputsMultipleOriginalOutputsFails() { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3)); PCollection<Integer> output = input.apply("Map", MapElements.via(fn)); PCollection<Integer> reappliedOutput = input.apply("ReMap", MapElements.via(fn)); thrown.expect(IllegalArgumentException.class); factory.mapOutputs( PCollectionList.of(output).and(input).and(reappliedOutput).expand(), reappliedOutput); } }
@Override public PDone expand(PCollection<Iterable<T>> input) { input .apply("GroupGlobally", new GroupGlobally<>(rewindowingStrategy)) .apply("GetPane", MapElements.via(paneExtractor)) .setCoder(IterableCoder.of(input.getCoder())) .apply("RunChecks", ParDo.of(new SingletonCheckerDoFn<>(checkerFn, site))) .apply("VerifyAssertions", new DefaultConcludeTransform()); return PDone.in(input.getPipeline()); } }
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(Options.class); options.setRunner(FlinkRunner.class); Pipeline p = Pipeline.create(options); p.apply("ReadLines", TextIO.Read.from(options.getInput())) .apply(new CountWords()) .apply(MapElements.via(new FormatAsTextFn())) .apply("WriteCounts", TextIO.Write.to(options.getOutput())); p.run(); }
public static void main(String[] args) { WordCountOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(WordCountOptions.class); Pipeline p = Pipeline.create(options); p.apply("ReadLines", TextIO.read().from(options.getInputFile())) .apply(new CountWords()) .apply(MapElements.via(new FormatAsTextFn())) .apply("WriteCounts", TextIO.write().to(options.getOutput())); p.run(); } }