/** * Creates a compression stream without any additional wrapping into * buffering streams. */ public CompressionOutputStream createPlainCompressionStream( OutputStream downStream, Compressor compressor) throws IOException { CompressionCodec codec = getCodec(conf); ((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024); return codec.createOutputStream(downStream, compressor); }
@Override public CompressedSliceOutput get() { try { compressor.reset(); compressedOutput.reset(); CompressionOutputStream compressionStream = codec.createOutputStream(compressedOutput, compressor); return new CompressedSliceOutput(compressionStream, compressedOutput, this, () -> {}); } catch (IOException e) { throw new UncheckedIOException(e); } } }
public static OutputStream makePathAndOutputStream(JobContext job, Path outputPath, boolean deleteExisting) throws IOException { OutputStream retVal; FileSystem fs = outputPath.getFileSystem(job.getConfiguration()); Class<? extends CompressionCodec> codecClass; CompressionCodec codec = null; if (FileOutputFormat.getCompressOutput(job)) { codecClass = FileOutputFormat.getOutputCompressorClass(job, GzipCodec.class); codec = ReflectionUtils.newInstance(codecClass, job.getConfiguration()); outputPath = new Path(outputPath + codec.getDefaultExtension()); } if (fs.exists(outputPath)) { if (deleteExisting) { fs.delete(outputPath, false); } else { throw new ISE("outputPath[%s] must not exist.", outputPath); } } if (FileOutputFormat.getCompressOutput(job)) { retVal = codec.createOutputStream(fs.create(outputPath, false)); } else { retVal = fs.create(outputPath, false); } return retVal; }
@Override public CompressedSliceOutput get() { try { compressor.reset(); bufferedOutput.reset(); CompressionOutputStream compressionStream = codec.createOutputStream(bufferedOutput, compressor); return new CompressedSliceOutput(compressionStream, bufferedOutput, this, () -> CodecPool.returnCompressor(compressor)); } catch (IOException e) { throw new UncheckedIOException(e); } } }
public FlexibleDelimitedFileWriter(LogFilePath path, CompressionCodec codec) throws IOException { Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mCountingStream = new CountingOutputStream(fs.create(fsPath)); this.mWriter = (codec == null) ? new BufferedOutputStream( this.mCountingStream) : new BufferedOutputStream( codec.createOutputStream(this.mCountingStream, mCompressor = CodecPool.getCompressor(codec))); }
public DelimitedTextFileWriter(LogFilePath path, CompressionCodec codec) throws IOException { Path fsPath = new Path(path.getLogFilePath()); FileSystem fs = FileUtil.getFileSystem(path.getLogFilePath()); this.mCountingStream = new CountingOutputStream(fs.create(fsPath)); this.mWriter = (codec == null) ? new BufferedOutputStream( this.mCountingStream) : new BufferedOutputStream( codec.createOutputStream(this.mCountingStream, mCompressor = CodecPool.getCompressor(codec))); }
@Override public synchronized OutputStream createCompressionStream( OutputStream downStream, Compressor compressor, int downStreamBufferSize) throws IOException { if (!isSupported()) { throw new IOException( "LZO codec class not specified. Did you forget to set property " + CONF_LZO_CLASS + "?"); } OutputStream bos1 = null; if (downStreamBufferSize > 0) { bos1 = new BufferedOutputStream(downStream, downStreamBufferSize); } else { bos1 = downStream; } conf.setInt(IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY, IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_DEFAULT); CompressionOutputStream cos = codec.createOutputStream(bos1, compressor); BufferedOutputStream bos2 = new BufferedOutputStream(new FinishOnFlushCompressionStream(cos), DATA_OBUF_SIZE); return bos2; } },
/** * Convert an output stream to a compressed output stream based on codecs codecs in the Job * Configuration. Caller specifies directly whether file is compressed or not * * @param jc * Job Configuration * @param out * Output Stream to be converted into compressed output stream * @param isCompressed * whether the output stream needs to be compressed or not * @return compressed output stream */ public static OutputStream createCompressedStream(JobConf jc, OutputStream out, boolean isCompressed) throws IOException { if (isCompressed) { Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc); return codec.createOutputStream(out); } else { return (out); } }
/** * Convert an output stream to a compressed output stream based on codecs codecs in the Job * Configuration. Caller specifies directly whether file is compressed or not * * @param jc * Job Configuration * @param out * Output Stream to be converted into compressed output stream * @param isCompressed * whether the output stream needs to be compressed or not * @return compressed output stream */ public static OutputStream createCompressedStream(JobConf jc, OutputStream out, boolean isCompressed) throws IOException { if (isCompressed) { Class<? extends CompressionCodec> codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class); CompressionCodec codec = ReflectionUtil.newInstance(codecClass, jc); return codec.createOutputStream(out); } else { return (out); } }
public class MultiSplitRecordWriter<K, V> implements RecordWriter<K, V> { private boolean ignoreSeparatorOnNull; protected NewDataOutputStream out; private String keyValueSeparator; LineRecordWriter<K, V> writer; public MultiSplitRecordWriter(NewDataOutputStream out, String keyValueSeparator, boolean ignoreSeparatorOnNull) { this.out = out; this.keyValueSeparator = keyValueSeparator; this.ignoreSeparatorOnNull = ignoreSeparatorOnNull; this.writer = new LineRecordWriter<K, V>(out, keyValueSeparator); } public synchronized void write(K key, V value) throws IOException { if (splitSize < out.getSize()) { // writer.flush(); writer.close(null); out = new NewDataOutputStream(codec.createOutputStream(createFile())); writer = new LineRecordWriter<K, V>(out, keyValueSeparator); } writer.write(key, value); } public synchronized void close(Reporter reporter) throws IOException { out.close(); writer.close(reporter); } }
/** * Create an output stream with a codec taken from the global CodecPool. * * @param codec The codec to use to create the output stream. * @param conf The configuration to use if we need to create a new codec. * @param out The output stream to wrap. * @return The new output stream * @throws IOException */ static CompressionOutputStream createOutputStreamWithCodecPool( CompressionCodec codec, Configuration conf, OutputStream out) throws IOException { Compressor compressor = CodecPool.getCompressor(codec, conf); CompressionOutputStream stream = null; try { stream = codec.createOutputStream(out, compressor); } finally { if (stream == null) { CodecPool.returnCompressor(compressor); } else { stream.setTrackedCompressor(compressor); } } return stream; }
compressor = CodecPool.getCompressor(codec, conf); cmpOut = codec.createOutputStream(fsOut, compressor); serializer = EventSerializerFactory.getInstance(serializerType, serializerContext, cmpOut);
java.io.InputStream in = null; try { out = codec.createOutputStream( new java.io.FileOutputStream(args[i])); byte[] buffer = new byte[100];
public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { boolean ignoreSeparatorOnNull = job.getBoolean("mapred.textoutputformat.ignore.separator", false); String keyValueSeparator = job.get("mapred.textoutputformat.separator", "\t"); splitSize = job.getLong(MR_REDUCE_MAX_FILE_PER_FILE, SPLIT_SIZE); jobConf = job; fileName = name; jobProgress = progress; Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job, GzipCodec.class); // create the named codec codec = ReflectionUtils.newInstance(codecClass, job); FSDataOutputStream fileOut = createFile(); return new MultiSplitRecordWriter<K, V>(new NewDataOutputStream(codec.createOutputStream(fileOut)), keyValueSeparator, ignoreSeparatorOnNull); }
ReflectionUtils.setConf(this.codec, this.conf); this.compressor = CodecPool.getCompressor(this.codec); this.deflateFilter = this.codec.createOutputStream(buffer, compressor); this.deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
private void encodeCellsTo(OutputStream os, CellScanner cellScanner, Codec codec, CompressionCodec compressor) throws IOException { Compressor poolCompressor = null; try { if (compressor != null) { if (compressor instanceof Configurable) { ((Configurable) compressor).setConf(this.conf); } poolCompressor = CodecPool.getCompressor(compressor); os = compressor.createOutputStream(os, poolCompressor); } Codec.Encoder encoder = codec.getEncoder(os); while (cellScanner.advance()) { encoder.write(cellScanner.current()); } encoder.flush(); } catch (BufferOverflowException | IndexOutOfBoundsException e) { throw new DoNotRetryIOException(e); } finally { os.close(); if (poolCompressor != null) { CodecPool.returnCompressor(poolCompressor); } } }
CompressionOutputStream deflateFilter = codec.createOutputStream(compressedDataBuffer);
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
compressor = CodecPool.getCompressor(codec); valueBuffer = new NonSyncDataOutputBuffer(); deflateFilter = codec.createOutputStream(valueBuffer, compressor); deflateOut = new DataOutputStream(deflateFilter);
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }