mobFilesChecked.addAndGet(1); if (r != null) { r.close(true);
hfilesChecked.addAndGet(1); if (r != null) { r.close(true);
/** * Indicate that the scanner has finished reading with this reader. We need to decrement the ref * count, and also, if this is not the common pread reader, we should close it. */ void readCompleted() { refCount.decrementAndGet(); if (!shared) { try { reader.close(false); if (this.listener != null) { this.listener.storeFileReaderClosed(this); } } catch (IOException e) { LOG.warn("failed to close stream reader", e); } } }
assertEquals("k", toRowStr(scanner.getCell())); reader.close(); deleteTestDir(fs);
scanner.seekTo(CellUtil.createCell(keys.get(i))); reader.close();
cryptoContext.getKeyBytes())); } finally { reader.close();
assertEquals(1, blockIndexReader.rootBlockContainingKey( toKV("l", tagUsage))); reader.close(); deleteTestDir(fs);
public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFileContext context = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withFileContext(context) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf); try { reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); // position to the start of file // Scanner does not do Cells yet. Do below for now till fixed. cc = scanner.getCell(); if (CellComparator.getInstance().compareRows(c, cc) != 0) { throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString()); } } finally { reader.close(); } }
/** * Verify that compression and data block encoding are respected by the * Store.createWriterInTmp() method, used on store flush. */ @Test public void testCreateWriter() throws Exception { Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(conf); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(family) .setCompressionType(Compression.Algorithm.GZ).setDataBlockEncoding(DataBlockEncoding.DIFF) .build(); init(name.getMethodName(), conf, hcd); // Test createWriterInTmp() StoreFileWriter writer = store.createWriterInTmp(4, hcd.getCompressionType(), false, true, false, false); Path path = writer.getPath(); writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1))); writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2))); writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3))); writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4))); writer.close(); // Verify that compression and encoding settings are respected HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), true, conf); assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm()); assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding()); reader.close(); }
private void metablocks(final String compress) throws Exception { Path mFile = new Path(ROOT_DIR, "meta.hfile"); FSDataOutputStream fout = createFSOutput(mFile); HFileContext meta = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(compress)) .withBlockSize(minBlockSize).build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) .create(); someTestingWithMetaBlock(writer); writer.close(); fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); reader.loadFileInfo(); // No data -- this should return false. assertFalse(reader.getScanner(false, false).seekTo()); someReadingWithMetaBlock(reader); fs.delete(mFile, true); reader.close(); fin.close(); }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; } }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; }
protected void testSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); HFileScanner scanner = reader.getScanner(false, true); // lies before the start of the file. assertEquals(-1, scanner.seekTo(toKV("a", tagUsage))); assertEquals(1, scanner.seekTo(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getCell())); // Across a block boundary now. // 'h' does not exist so we will get a '1' back for not found. assertEquals(0, scanner.seekTo(toKV("i", tagUsage))); assertEquals("i", toRowStr(scanner.getCell())); assertEquals(1, scanner.seekTo(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getCell())); reader.close(); deleteTestDir(fs); }
private static byte[] extractHFileKey(Path path) throws Exception { HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path, new CacheConfig(conf), true, conf); try { reader.loadFileInfo(); Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext(); assertNotNull("Reader has a null crypto context", cryptoContext); Key key = cryptoContext.getKey(); if (key == null) { return null; } return key.getEncoded(); } finally { reader.close(); } }
private int verifyHFile(Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); assertTrue(count > 0); reader.close(); return count; }
@Override void tearDown() throws Exception { reader.close(); }
private static byte[] extractHFileKey(Path path) throws Exception { HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path, new CacheConfig(conf), true, conf); try { reader.loadFileInfo(); Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext(); assertNotNull("Reader has a null crypto context", cryptoContext); Key key = cryptoContext.getKey(); assertNotNull("Crypto context has no key", key); return key.getEncoded(); } finally { reader.close(); } }
private byte[] extractHFileKey(Path path) throws Exception { HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path, new CacheConfig(conf), true, conf); try { reader.loadFileInfo(); Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext(); assertNotNull("Reader has a null crypto context", cryptoContext); Key key = cryptoContext.getKey(); assertNotNull("Crypto context has no key", key); return key.getEncoded(); } finally { reader.close(); } }
public void close(boolean evictOnClose) throws IOException { synchronized (this) { if (closed) { return; } reader.close(evictOnClose); closed = true; } if (listener != null) { listener.storeFileReaderClosed(this); } }
@Override public void close() throws IOException { if (in != null) { in.close(); in = null; } } }