/** @deprecated Use {@link #withCompression}. */ @Deprecated public Read withCompressionType(TextIO.CompressionType compressionType) { return withCompression(compressionType.canonical); }
@Test public void testInitialSplitGzipModeTxt() throws Exception { PipelineOptions options = TestPipeline.testingPipelineOptions(); long desiredBundleSize = 1000; File largeTxt = writeToFile(LARGE, tempFolder, "large.txt", UNCOMPRESSED); // Sanity check: file is at least 2 bundles long. assertThat(largeTxt.length(), greaterThan(2 * desiredBundleSize)); FileBasedSource<String> source = TextIO.read().from(largeTxt.getPath()).withCompression(GZIP).getSource(); List<? extends FileBasedSource<String>> splits = source.split(desiredBundleSize, options); // Exactly 1 split, even though splittable text file, since using GZIP mode. assertThat(splits, hasSize(equalTo(1))); SourceTestUtils.assertSourcesEqualReferenceSource(source, splits, options); }
/** * Helper method that runs a variety of ways to read a single file using TextIO and checks that * they all match the given expected output. * * <p>The transforms being verified are: * * <ul> * <li>TextIO.read().from(filename).withCompression(compressionType) * <li>TextIO.read().from(filename).withCompression(compressionType) .withHintMatchesManyFiles() * <li>TextIO.readAll().withCompression(compressionType) * </ul> */ private static void assertReadingCompressedFileMatchesExpected( File file, Compression compression, List<String> expected, Pipeline p) { TextIO.Read read = TextIO.read().from(file.getPath()).withCompression(compression); PAssert.that(p.apply("Read_" + file + "_" + compression.toString(), read)) .containsInAnyOrder(expected); PAssert.that( p.apply( "Read_" + file + "_" + compression.toString() + "_many", read.withHintMatchesManyFiles())) .containsInAnyOrder(expected); TextIO.ReadAll readAll = TextIO.readAll().withCompression(compression); PAssert.that( p.apply("Create_" + file, Create.of(file.getPath())) .apply("Read_" + compression.toString(), readAll)) .containsInAnyOrder(expected); }
@Test public void testReadDisplayData() { TextIO.Read read = TextIO.read().from("foo.*").withCompression(BZIP2); DisplayData displayData = DisplayData.from(read); assertThat(displayData, hasDisplayItem("filePattern", "foo.*")); assertThat(displayData, hasDisplayItem("compressionType", BZIP2.toString())); }
@Test public void testCompressionIsSet() throws Exception { TextIO.Read read = TextIO.read().from("/tmp/test"); assertEquals(AUTO, read.getCompression()); read = TextIO.read().from("/tmp/test").withCompression(GZIP); assertEquals(GZIP, read.getCompression()); }