LongWritable position = new LongWritable(); long nextBlock = pos; boolean blockCompressed = dataReader.isBlockCompressed(); while(dataReader.next(key, value)) { if (blockCompressed) {
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
else if (reader.isBlockCompressed()) compressionType = CompressionType.BLOCK; else
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = (FileSplit) split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.initIOContext(blockStart, blockPointer, split.getPath().toString()); } }
/** * Clones the attributes (like compression of the input file and creates a * corresponding Writer * @param inputFile the path of the input file whose attributes should be * cloned * @param outputFile the path of the output file * @param prog the Progressable to report status during the file write * @return Writer * @throws IOException */ public Writer cloneFileAttributes(Path inputFile, Path outputFile, Progressable prog) throws IOException { FileSystem srcFileSys = inputFile.getFileSystem(conf); Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true); boolean compress = reader.isCompressed(); boolean blockCompress = reader.isBlockCompressed(); CompressionCodec codec = reader.getCompressionCodec(); reader.close(); Writer writer = createWriter(outputFile.getFileSystem(conf), conf, outputFile, keyClass, valClass, compress, blockCompress, codec, prog, new Metadata()); return writer; }
public static void read(Path inputPath) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); SequenceFile.Reader reader = new SequenceFile.Reader(fs, inputPath, conf); HBaseScanAvroStock.AvroStockReader stockReader = new HBaseScanAvroStock.AvroStockReader(); try { System.out.println( "Is block compressed = " + reader.isBlockCompressed()); ImmutableBytesWritable key = new ImmutableBytesWritable(); Result value = new Result(); while (reader.next(key, value)) { Stock stock = stockReader.decode(value.getValue( STOCK_DETAILS_COLUMN_FAMILY_AS_BYTES, STOCK_COLUMN_QUALIFIER_AS_BYTES)); System.out.println(new String(key.get()) + ": " + ToStringBuilder .reflectionToString(stock, ToStringStyle.SIMPLE_STYLE)); } } finally { reader.close(); } } }
public static void read(Path inputPath) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); SequenceFile.Reader reader = //<co id="ch03_comment_seqfile_read1"/> new SequenceFile.Reader(fs, inputPath, conf); try { System.out.println( "Is block compressed = " + reader.isBlockCompressed()); Text key = new Text(); IntWritable value = new IntWritable(); while (reader.next(key, value)) { //<co id="ch03_comment_seqfile_read2"/> System.out.println(key + "," + value); } } finally { reader.close(); } } }
public static void read(Path inputPath) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); SequenceFile.Reader reader = //<co id="ch03_comment_seqfile_read1"/> new SequenceFile.Reader(fs, inputPath, conf); try { System.out.println( "Is block compressed = " + reader.isBlockCompressed()); Text key = new Text(); StockPriceWritable value = new StockPriceWritable(); while (reader.next(key, value)) { //<co id="ch03_comment_seqfile_read2"/> System.out.println(key + "," + value); } } finally { reader.close(); } } }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
/** * Returns true if records are block-compressed. */ public boolean isBlockCompressed() { return innerReader.isBlockCompressed(); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
/** * Returns true if records are block-compressed. */ public boolean isBlockCompressed() { return innerReader.isBlockCompressed(); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }
@SuppressWarnings("unchecked") public void put(SegmentDescriptor stream) throws IOException { if (size() == 0) { compress = stream.in.isCompressed(); blockCompress = stream.in.isBlockCompressed(); } else if (compress != stream.in.isCompressed() || blockCompress != stream.in.isBlockCompressed()) { throw new IOException("All merged files must be compressed or not."); } super.put(stream); }