BlockRegion region = e.getValue().getRegion(); String blkCompression = e.getValue().getCompressionAlgorithm().getName(); out.printf("%s %s %s %s %s%n", Align.format(blkName, nameWidth, Align.LEFT), Align.format(region.getOffset(), offsetWidth,
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
static String[] getSupportedAlgorithms() { Algorithm[] algos = Algorithm.class.getEnumConstants(); ArrayList<String> ret = new ArrayList<String>(); for (Algorithm a : algos) { if (a.isSupported()) { ret.add(a.getName()); } } return ret.toArray(new String[ret.size()]); } }
Decompressor decompressor = compressName.getDecompressor(); FileContext fileContext = FileContext.getFileContext( thisNodeFile.getPath().toUri(), conf); in = compressName.createDecompressionStream( new BoundedRangeFileInputStream(fsin, candidate.getStartIndex(), } catch (IOException e) { System.err.println(e.getMessage()); compressName.returnDecompressor(decompressor); continue; } finally {
Algorithm compressName = Compression.getCompressionAlgorithmByName( compressAlgo); Decompressor decompressor = compressName.getDecompressor(); FileContext fileContext = FileContext.getFileContext( thisNodeFile.getPath().toUri(), conf); InputStream in = null; try { in = compressName.createDecompressionStream( new BoundedRangeFileInputStream(fsin, candidate.getStartIndex(), candidate.getFileCompressedSize()), decompressor,
public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize, String compress, int minBlkSize) throws IOException { Configuration conf = new Configuration(); conf.setBoolean("hadoop.native.lib", true); CompressionCodec codec = null; if ("lzo".equals(compress)) { codec = Compression.Algorithm.LZO.getCodec(); } else if ("gz".equals(compress)) { codec = Compression.Algorithm.GZ.getCodec(); } else if (!"none".equals(compress)) throw new IOException("Codec not supported."); this.fsdos = fs.create(path, true, osBufferSize); if (!"none".equals(compress)) { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK, codec); } else { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.NONE, null); } }
public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize, String compress, int minBlkSize) throws IOException { Configuration conf = new Configuration(); conf.setBoolean("hadoop.native.lib", true); CompressionCodec codec = null; if ("lzo".equals(compress)) { codec = Compression.Algorithm.LZO.getCodec(); } else if ("gz".equals(compress)) { codec = Compression.Algorithm.GZ.getCodec(); } else if (!"none".equals(compress)) throw new IOException("Codec not supported."); this.fsdos = fs.create(path, true, osBufferSize); if (!"none".equals(compress)) { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK, codec); } else { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.NONE, null); } }
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize, String compress, int minBlkSize) throws IOException { Configuration conf = new Configuration(); conf.setBoolean("hadoop.native.lib", true); CompressionCodec codec = null; if ("lzo".equals(compress)) { codec = Compression.Algorithm.LZO.getCodec(); } else if ("gz".equals(compress)) { codec = Compression.Algorithm.GZ.getCodec(); } else if (!"none".equals(compress)) throw new IOException("Codec not supported."); this.fsdos = fs.create(path, true, osBufferSize); if (!"none".equals(compress)) { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.BLOCK, codec); } else { writer = SequenceFile.createWriter(conf, fsdos, BytesWritable.class, BytesWritable.class, SequenceFile.CompressionType.NONE, null); } }
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
public Decompressor getDecompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Decompressor decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { if (decompressor.finished()) { // Somebody returns the decompressor to CodecPool but is still using // it. LOG.warn("Deompressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a decompressor: " + decompressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * decompressor is referenced after returned back to the codec pool. */ decompressor.reset(); } return decompressor; } return null; }
/** * @param compressionAlgo * The compression algorithm to be used to for compression. * @throws IOException */ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.fsOut = fsOut; this.posStart = fsOut.getPos(); fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf)); this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes()); this.compressor = compressAlgo.getCompressor(); try { this.out = compressionAlgo.createCompressionStream(fsBufferedOutput, compressor, 0); } catch (IOException e) { compressAlgo.returnCompressor(compressor); throw e; } }
public RBlockState(Algorithm compressionAlgo, FSDataInputStream fsin, BlockRegion region, Configuration conf, Reader r) throws IOException { this.compressAlgo = compressionAlgo; Decompressor decompressor = compressionAlgo.getDecompressor(); this.region = region; try { InputStream in = compressAlgo.createDecompressionStream(new BoundedRangeFileInputStream(fsin, region.getOffset(), region.getCompressedSize()), decompressor, DTFile.getFSInputBufferSize(conf)); int l = 1; r.baos.reset(); byte[] buf = new byte[DTFile.getFSInputBufferSize(conf)]; while (l >= 0) { l = in.read(buf); if (l > 0) { r.baos.write(buf, 0, l); } } // keep decompressed data into cache byte[] blockData = r.baos.toByteArray(); rbain = new ReusableByteArrayInputStream(blockData); } catch (IOException e) { compressAlgo.returnDecompressor(decompressor); throw e; } }
/** * @param compressionAlgo * The compression algorithm to be used to for compression. * @throws IOException */ public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut, BytesWritable fsOutputBuffer, Configuration conf) throws IOException { this.compressAlgo = compressionAlgo; this.fsOut = fsOut; this.posStart = fsOut.getPos(); fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf)); this.fsBufferedOutput = new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.get()); this.compressor = compressAlgo.getCompressor(); try { this.out = compressionAlgo.createCompressionStream(fsBufferedOutput, compressor, 0); } catch (IOException e) { compressAlgo.returnCompressor(compressor); throw e; } }