@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
uuid = uuidText.toString(); if (!reader.isCompressed()) compressionType = CompressionType.NONE; else if (reader.isBlockCompressed())
uuid = uuidText.toString(); if (!reader.isCompressed()) compressionType = CompressionType.NONE; else if (reader.isBlockCompressed())
job.setOutputKeyClass(NullWritable.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); if (seqReader.isCompressed()) { log.info("SEQ Compression Codec {}", seqReader.getCompressionCodec().toString()); log.info("SEQ Compression Type {}", seqReader.getCompressionType().toString());
isCompressed = in.isCompressed(); isBlockCompressed = in.isBlockCompressed(); codec = in.getCompressionCodec();
isCompressed = in.isCompressed(); isBlockCompressed = in.isBlockCompressed(); codec = in.getCompressionCodec();
assertEquals("is reduce output compressed " + output, redCompression != CompressionType.NONE, rdr.isCompressed()); rdr.close(); } finally {
assertEquals("is reduce output compressed " + output, redCompression != CompressionType.NONE, rdr.isCompressed()); rdr.close(); } finally {
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
/** * Returns true if values are compressed. */ public boolean isCompressed() { return innerReader.isCompressed(); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
/** * Returns true if values are compressed. */ public boolean isCompressed() { return innerReader.isCompressed(); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }