private boolean initializeCompressionContext(Configuration conf, Path path) throws IOException { boolean doCompress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); if (doCompress) { try { this.compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), conf.getBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true)); } catch (Exception e) { throw new IOException("Failed to initiate CompressionContext", e); } } return doCompress; }
@Override public void init(FileSystem fs, Path path, Configuration conf, FSDataInputStream stream) throws IOException { this.conf = conf; this.path = path; this.fs = fs; this.fileLength = this.fs.getFileStatus(path).getLen(); String cellCodecClsName = initReader(stream); boolean compression = hasCompression(); if (compression) { // If compression is enabled, new dictionaries are created here. try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), hasTagCompression()); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } initAfterCompression(cellCodecClsName); }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
@SuppressWarnings("unused") @Test public void testConstructorsArePresent() throws Exception { // "testing" via the presence of these constructors IndexedWALEditCodec codec1 = new IndexedWALEditCodec(); IndexedWALEditCodec codec2 = new IndexedWALEditCodec(new Configuration(false), new CompressionContext(LRUDictionary.class, false, false)); } }
@Test public void testWithCompression() throws Exception { // get the FS ready to read/write the edit Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withCompression"); Path testFile = new Path(testDir, "testfile"); FileSystem fs = UTIL.getTestFileSystem(); List<WALEdit> edits = getEdits(); CompressionContext compression = new CompressionContext(LRUDictionary.class, false, false); writeReadAndVerify(compression, fs, edits, testFile); }
@Test public void testWithCompression() throws Exception { // get the FS ready to read/write the edit Path testDir = UTIL.getDataTestDir("TestReadWriteCustomEdits_withCompression"); Path testFile = new Path(testDir, "testfile"); FileSystem fs = UTIL.getTestFileSystem(); List<WALEdit> edits = getEdits(); CompressionContext compression = new CompressionContext(LRUDictionary.class); WALEditCodec codec = WALEditCodec.create(UTIL.getConfiguration(), compression); writeReadAndVerify(codec, fs, edits, testFile); }
public boolean initializeCompressionContext(Configuration conf, Path path) throws IOException { boolean doCompress = conf.getBoolean(HConstants.ENABLE_WAL_COMPRESSION, false); if (doCompress) { try { this.compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), conf.getBoolean( CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, true)); } catch (Exception e) { throw new IOException("Failed to initiate CompressionContext", e); } } return doCompress; }
@Override public void init(FileSystem fs, Path path, Configuration conf) throws IOException { this.conf = conf; this.path = path; reader = new WALReader(fs, path, conf); this.fs = fs; // If compression is enabled, new dictionaries are created here. boolean compression = reader.isWALCompressionEnabled(); if (compression) { try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } }
try { if (this.compressionContext == null) { this.compressionContext = new CompressionContext(LRUDictionary.class); } else { this.compressionContext.clear();
@Override public void init(FileSystem fs, Path path, Configuration conf, FSDataInputStream stream) throws IOException { this.conf = conf; this.path = path; this.fs = fs; this.fileLength = this.fs.getFileStatus(path).getLen(); String cellCodecClsName = initReader(stream); boolean compression = hasCompression(); if (compression) { // If compression is enabled, new dictionaries are created here. try { if (compressionContext == null) { compressionContext = new CompressionContext(LRUDictionary.class, FSUtils.isRecoveredEdits(path), hasTagCompression()); } else { compressionContext.clear(); } } catch (Exception e) { throw new IOException("Failed to initialize CompressionContext", e); } } initAfterCompression(cellCodecClsName); }
private void doTest(boolean compressTags, boolean offheapKV) throws Exception { Configuration conf = new Configuration(false); conf.setBoolean(CompressionContext.ENABLE_WAL_TAGS_COMPRESSION, compressTags); WALCellCodec codec = new WALCellCodec(conf, new CompressionContext(LRUDictionary.class, false, compressTags)); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);