reader.getCompressionType(), reader.getCompressionCodec());
/** * @return the compression codec in use for this file. */ public Class<? extends CompressionCodec> getCompressionCodecClass() { if (!isReader) return DefaultCodec.class; Reader reader = null; try { reader = createReader(); if (reader.getCompressionCodec() == null) return null; return reader.getCompressionCodec().getClass(); } catch (final Exception e) { throw new RuntimeException(e); } finally { if (reader != null) try { reader.close(); } catch (final IOException e1) { } } }
codec = in.getCompressionCodec();
codec = in.getCompressionCodec();
assertEquals(codec.getClass(), reader.getCompressionCodec().getClass());
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { Reader reader = new Reader(conf, Reader.file(inputFile), new Reader.OnlyHeaderOption()); CompressionType compress = reader.getCompressionType(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(conf, Writer.file(outputFile), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compress, codec), Writer.progressable(prog)); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { Reader reader = new Reader(conf, Reader.file(inputFile), new Reader.OnlyHeaderOption()); CompressionType compress = reader.getCompressionType(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(conf, Writer.file(outputFile), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compress, codec), Writer.progressable(prog)); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { Reader reader = new Reader(conf, Reader.file(inputFile), new Reader.OnlyHeaderOption()); CompressionType compress = reader.getCompressionType(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(conf, Writer.file(outputFile), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compress, codec), Writer.progressable(prog)); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { Reader reader = new Reader(conf, Reader.file(inputFile), new Reader.OnlyHeaderOption()); CompressionType compress = reader.getCompressionType(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(conf, Writer.file(outputFile), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compress, codec), Writer.progressable(prog)); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
/** * @return the compression codec in use for this file. */ public Class<? extends CompressionCodec> getCompressionCodecClass() { if (!isReader) return DefaultCodec.class; Reader reader = null; try { reader = createReader(); if (reader.getCompressionCodec() == null) return null; return reader.getCompressionCodec().getClass(); } catch (final Exception e) { throw new RuntimeException(e); } finally { if (reader != null) try { reader.close(); } catch (final IOException e1) { } } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_compressed() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); try (ModelOutput<StringOption> out = format.codec(new DefaultCodec()) .createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), instanceOf(DefaultCodec.class)); } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_compressed_conf() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, DefaultCodec.class.getName()); try (ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), instanceOf(DefaultCodec.class)); } }
/** * invalid compressed output. * @throws Exception if failed */ @Test public void output_compressed_invalid() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing").toURI()); format.getConf().set(SequenceFileFormat.KEY_COMPRESSION_CODEC, "__INVALID__"); try (ModelOutput<StringOption> out = format.createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), is(nullValue())); } }
/** * compressed output. * @throws Exception if failed */ @Test public void output_no_compressed() throws Exception { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(folder.newFile("testing.gz").toURI()); try (ModelOutput<StringOption> out = format.codec(null) .createOutput(StringOption.class, fs, path, new Counter())) { out.write(new StringOption("Hello, world!")); } try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(fs.makeQualified(path)))) { assertThat(reader.getCompressionCodec(), is(nullValue())); } }
/** * Returns the compression codec of data in this file. */ public CompressionCodec getCompressionCodec() { return innerReader.getCompressionCodec(); }
/** * Returns the compression codec of data in this file. */ public CompressionCodec getCompressionCodec() { return innerReader.getCompressionCodec(); }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { Reader reader = new Reader(conf, Reader.file(inputFile), new Reader.OnlyHeaderOption()); CompressionType compress = reader.getCompressionType(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(conf, Writer.file(outputFile), Writer.keyClass(keyClass), Writer.valueClass(valClass), Writer.compression(compress, codec), Writer.progressable(prog)); return writer; }