private void runTestWrite(String[] elems, String header, String footer, int numShards) throws Exception { String outputName = "file.txt"; Path baseDir = Files.createTempDirectory(tempFolder.getRoot().toPath(), "testwrite"); ResourceId baseFilename = FileBasedSink.convertToFileResourceIfPossible(baseDir.resolve(outputName).toString()); PCollection<String> input = p.apply("CreateInput", Create.of(Arrays.asList(elems)).withCoder(StringUtf8Coder.of())); TextIO.TypedWrite<String, Void> write = TextIO.write().to(baseFilename).withHeader(header).withFooter(footer).withOutputFilenames(); if (numShards == 1) { write = write.withoutSharding(); } else if (numShards > 0) { write = write.withNumShards(numShards).withShardNameTemplate(ShardNameTemplate.INDEX_OF_MAX); } input.apply(write); p.run(); assertOutputFiles( elems, header, footer, numShards, baseFilename, firstNonNull( write.getShardTemplate(), DefaultFilenamePolicy.DEFAULT_UNWINDOWED_SHARD_TEMPLATE)); }
TextIO.write() .to(options.getOutputFailureFile()) .withHeader("Filename,Error") .withoutSharding());
TextIO.write() .to(options.getOutputFailureFile()) .withHeader("Filename,Error") .withoutSharding());
@Test public void testWriteDisplayData() { TextIO.Write write = TextIO.write() .to("/foo") .withSuffix("bar") .withShardNameTemplate("-SS-of-NN-") .withNumShards(100) .withFooter("myFooter") .withHeader("myHeader"); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("filePrefix", "/foo")); assertThat(displayData, hasDisplayItem("fileSuffix", "bar")); assertThat(displayData, hasDisplayItem("fileHeader", "myHeader")); assertThat(displayData, hasDisplayItem("fileFooter", "myFooter")); assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-")); assertThat(displayData, hasDisplayItem("numShards", 100)); assertThat(displayData, hasDisplayItem("writableByteChannelFactory", "UNCOMPRESSED")); }
@Test public void testWriteDisplayDataValidateThenHeader() { TextIO.Write write = TextIO.write().to("foo").withHeader("myHeader"); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("fileHeader", "myHeader")); }