}; FileIO.Write<String, String> write = FileIO.<String, String>writeDynamic() .by( fn(
/** Writes to different file sinks based on a */ @Override public PDone expand(PCollection<FeatureRowExtended> input) { final String folderName = options.jobName != null ? options.jobName : "unknown-jobs"; FileIO.Write<String, FeatureRowExtended> write = FileIO.<String, FeatureRowExtended>writeDynamic() .by((rowExtended) -> rowExtended.getRow().getEntityName()) .withDestinationCoder(StringUtf8Coder.of()) .withNaming( Contextful.fn( (entityName) -> FileIO.Write.defaultNaming(folderName + "/" + entityName, suffix))) .via(Contextful.fn(toTextFunction), Contextful.fn((entityName) -> TextIO.sink())) .to(options.path); if (input.isBounded().equals(IsBounded.UNBOUNDED)) { Window<FeatureRowExtended> minuteWindow = Window.<FeatureRowExtended>into(FixedWindows.of(options.getWindowDuration())) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() .withAllowedLateness(Duration.ZERO); input = input.apply(minuteWindow); write = write.withNumShards(10); } WriteFilesResult<String> outputFiles = input.apply(write); return PDone.in(outputFiles.getPipeline()); } }
"Filenames should be resolved within a relative directory if '.to' is invoked", expected, getDefaultFileName(FileIO.writeDynamic().to("test").withNaming(o -> foo))); assertEquals( "Filenames should be resolved within a relative directory if '.to' is invoked", + "is not invoked", "foo", getDefaultFileName(FileIO.writeDynamic().withNaming(o -> foo))); assertEquals( "Filenames should be resolved as the direct result of the filenaming function if '.to' "
FileIO.<String, KV<String, String>>writeDynamic() .by(KV::getKey) .withDestinationCoder(StringUtf8Coder.of()) FileIO.<String, String>writeDynamic() .by(SerializableFunctions.constant("")) .withDestinationCoder(StringUtf8Coder.of())