@Test public void testSink() throws Exception { TextIO.Sink sink = TextIO.sink().withHeader("header").withFooter("footer"); File f = new File(tempFolder.getRoot(), "file"); try (WritableByteChannel chan = Channels.newChannel(new FileOutputStream(f))) { sink.open(chan); sink.write("a"); sink.write("b"); sink.write("c"); sink.flush(); } assertEquals(Arrays.asList("header", "a", "b", "c", "footer"), readLinesFromFile(f)); } }
pc1 = UnboundedWrite.ofDefaultWindow(pc1); WriteFilesResult results = pc1.apply(FileIO.<String> write().withNumShards(1).via(TextIO.sink()).to(path)); return PDone.in(results.getPipeline());
pc1 = UnboundedWrite.ofDefaultWindow(pc1); WriteFilesResult results = pc1.apply(FileIO.<String> write().withNumShards(1).via(TextIO.sink()).to(path)); return PDone.in(results.getPipeline());
@Test @Category(NeedsRunner.class) public void testWriteViaSink() throws Exception { List<String> data = ImmutableList.of("a", "b", "c", "d", "e", "f"); PAssert.that( p.apply(Create.of(data)) .apply( FileIO.<String>write() .to(tempFolder.getRoot().toString()) .withSuffix(".txt") .via(TextIO.sink()) .withIgnoreWindowing()) .getPerDestinationOutputFilenames() .apply(Values.create()) .apply(TextIO.readAll())) .containsInAnyOrder(data); p.run(); }
/** Writes to different file sinks based on a */ @Override public PDone expand(PCollection<FeatureRowExtended> input) { final String folderName = options.jobName != null ? options.jobName : "unknown-jobs"; FileIO.Write<String, FeatureRowExtended> write = FileIO.<String, FeatureRowExtended>writeDynamic() .by((rowExtended) -> rowExtended.getRow().getEntityName()) .withDestinationCoder(StringUtf8Coder.of()) .withNaming( Contextful.fn( (entityName) -> FileIO.Write.defaultNaming(folderName + "/" + entityName, suffix))) .via(Contextful.fn(toTextFunction), Contextful.fn((entityName) -> TextIO.sink())) .to(options.path); if (input.isBounded().equals(IsBounded.UNBOUNDED)) { Window<FeatureRowExtended> minuteWindow = Window.<FeatureRowExtended>into(FixedWindows.of(options.getWindowDuration())) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() .withAllowedLateness(Duration.ZERO); input = input.apply(minuteWindow); write = write.withNumShards(10); } WriteFilesResult<String> outputFiles = input.apply(write); return PDone.in(outputFiles.getPipeline()); } }
Contextful.of( tableManifestNaming, Requirements.requiresSideInputs(outputDirectoryName))) .via(Contextful.fn(KV::getValue), TextIO.sink()) .withTempDirectory(outputDir)); .by(SerializableFunctions.constant("")) .withDestinationCoder(StringUtf8Coder.of()) .via(TextIO.sink()) .withTempDirectory(outputDir) .withNaming(