@Override public Decoder getDecoder(ByteBuff buf) { return getDecoder(new ByteBuffInputStream(buf)); }
private void initAfterHeader0(boolean doCompress) throws IOException { WALCellCodec codec = getCodec(conf, this.compressionContext); this.cellEncoder = codec.getEncoder(getOutputStreamForCellEncoder()); if (doCompress) { this.compressor = codec.getByteStringCompressor(); } else { this.compressor = WALCellCodec.getNoneCompressor(); } }
protected WALCellCodec getCodec(Configuration conf, String cellCodecClsName, CompressionContext compressionContext) throws IOException { return WALCellCodec.create(conf, cellCodecClsName, compressionContext); }
@Override protected void initAfterCompression(String cellCodecClsName) throws IOException { WALCellCodec codec = getCodec(this.conf, cellCodecClsName, this.compressionContext); this.cellDecoder = codec.getDecoder(this.inputStream); if (this.hasCompression) { this.byteStringUncompressor = codec.getByteStringUncompressor(); } else { this.byteStringUncompressor = WALCellCodec.getNoneUncompressor(); } }
@Override protected void initAfterCompression(String cellCodecClsName) throws IOException { if (decryptor != null && cellCodecClsName.equals(SecureWALCellCodec.class.getName())) { WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, decryptor); this.cellDecoder = codec.getDecoder(this.inputStream); // We do not support compression with WAL encryption this.compressionContext = null; this.byteStringUncompressor = WALCellCodec.getNoneUncompressor(); this.hasCompression = false; } else { super.initAfterCompression(cellCodecClsName); } }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); Encoder encoder = codec.getEncoder(bos); if (offheapKV) { encoder.write(createOffheapKV(1)); Decoder decoder = codec.getDecoder(is); decoder.advance(); KeyValue kv = (KeyValue) decoder.current();
@Override public Encoder getEncoder(OutputStream os) { // compression isn't on, do the default thing if (this.compression == null) { return new IndexKeyValueEncoder(os); } // compression is on, return our one that will handle putting in the correct markers Encoder encoder = super.getEncoder(os); return new CompressedIndexKeyValueEncoder(os, encoder); }
/** * Create and setup a {@link WALCellCodec} from the * CompressionContext. * Cell Codec classname is read from {@link Configuration}. * Fully prepares the codec for use. * @param conf {@link Configuration} to read for the user-specified codec. If none is specified, * uses a {@link WALCellCodec}. * @param compression compression the codec should use * @return a {@link WALCellCodec} ready for use. * @throws UnsupportedOperationException if the codec cannot be instantiated */ public static WALCellCodec create(Configuration conf, CompressionContext compression) throws UnsupportedOperationException { String cellCodecClsName = getWALCellCodecClass(conf).getName(); return ReflectionUtils.instantiateWithCustomCtor(cellCodecClsName, new Class[] { Configuration.class, CompressionContext.class }, new Object[] { conf, compression }); }
Path testFile) throws IOException { WALCellCodec codec = WALCellCodec.create(UTIL.getConfiguration(), compressionContext); WALEdit edit = new WALEdit(); int numEdits = in.readInt(); edit.readFromCells(codec.getDecoder(in), numEdits); read.add(edit);
protected final void secureInitAfterHeader(boolean doCompress, Encryptor encryptor) throws IOException { if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false) && encryptor != null) { WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, encryptor); this.cellEncoder = codec.getEncoder(getOutputStreamForCellEncoder()); // We do not support compression this.compressionContext = null; this.compressor = WALCellCodec.getNoneCompressor(); } else { initAfterHeader0(doCompress); } }
@Override protected void initAfterCompression(String cellCodecClsName) throws IOException { WALCellCodec codec = getCodec(this.conf, cellCodecClsName, this.compressionContext); this.cellDecoder = codec.getDecoder(this.inputStream); if (this.hasCompression) { this.byteStringUncompressor = codec.getByteStringUncompressor(); } }
protected void initAfterHeader(boolean doCompress) throws IOException { WALCellCodec codec = getCodec(conf, this.compressionContext); this.cellEncoder = codec.getEncoder(this.output); if (doCompress) { this.compressor = codec.getByteStringCompressor(); } }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); Encoder encoder = codec.getEncoder(bos); if (offheapKV) { encoder.write(createOffheapKV(1)); Decoder decoder = codec.getDecoder(is); decoder.advance(); KeyValue kv = (KeyValue) decoder.current();
private void writeWALEdit(WALCellCodec codec, List<Cell> kvs, FSDataOutputStream out) throws IOException { out.writeInt(kvs.size()); Codec.Encoder cellEncoder = codec.getEncoder(out); // We interleave the two lists for code simplicity for (Cell kv : kvs) { cellEncoder.write(kv); } }
/** * Create and setup a {@link WALCellCodec} from the {@code cellCodecClsName} and * CompressionContext, if {@code cellCodecClsName} is specified. * Otherwise Cell Codec classname is read from {@link Configuration}. * Fully prepares the codec for use. * @param conf {@link Configuration} to read for the user-specified codec. If none is specified, * uses a {@link WALCellCodec}. * @param cellCodecClsName name of codec * @param compression compression the codec should use * @return a {@link WALCellCodec} ready for use. * @throws UnsupportedOperationException if the codec cannot be instantiated */ public static WALCellCodec create(Configuration conf, String cellCodecClsName, CompressionContext compression) throws UnsupportedOperationException { if (cellCodecClsName == null) { cellCodecClsName = getWALCellCodecClass(conf).getName(); } return ReflectionUtils.instantiateWithCustomCtor(cellCodecClsName, new Class[] { Configuration.class, CompressionContext.class }, new Object[] { conf, compression }); }
@Override public Decoder getDecoder(InputStream is) { // compression isn't enabled if (this.compression == null) { return useDefaultDecoder ? new IndexKeyValueDecoder(is) : new BinaryCompatibleIndexKeyValueDecoder(is); } // there is compression, so we get the standard decoder to handle reading those kvs Decoder decoder = super.getDecoder(is); // compression is on, reqturn our custom decoder return useDefaultDecoder ? new CompressedIndexKeyValueDecoder(is, decoder) : new BinaryCompatibleCompressedIndexKeyValueDecoder(is, decoder); }
@Override protected void initAfterHeader(boolean doCompress) throws IOException { if (conf.getBoolean(HConstants.ENABLE_WAL_ENCRYPTION, false) && encryptor != null) { WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, encryptor); this.cellEncoder = codec.getEncoder(this.output); // We do not support compression this.compressionContext = null; } else { super.initAfterHeader(doCompress); } }
String codecCls = WALCellCodec.getWALCellCodecClass(conf).getName(); try { TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
private WALCellCodec getCodec(Configuration conf, CompressionContext compressionContext) throws IOException { return WALCellCodec.create(conf, null, compressionContext); }
@Override protected void initAfterCompression(String cellCodecClsName) throws IOException { if (decryptor != null && cellCodecClsName.equals(SecureWALCellCodec.class.getName())) { WALCellCodec codec = SecureWALCellCodec.getCodec(this.conf, decryptor); this.cellDecoder = codec.getDecoder(this.inputStream); // We do not support compression with WAL encryption this.compressionContext = null; this.hasCompression = false; } else { super.initAfterCompression(cellCodecClsName); } }