/** * Opens the underlying reader. * It's not thread-safe. Use MobFileCache.openFile() instead. * @throws IOException */ public void open() throws IOException { sf.initReader(); }
private StoreFileReader createStreamReader(boolean canUseDropBehind) throws IOException { initReader(); StoreFileReader reader = fileInfo.open(this.fs, this.cacheConf, canUseDropBehind, -1L, primaryReplica, refCount, false); reader.copyFields(this.reader); return reader; }
/** * Validates a mob file by opening and closing it. * @param conf The current configuration. * @param fs The current file system. * @param path The path where the mob file is saved. * @param cacheConfig The current cache config. */ private static void validateMobFile(Configuration conf, FileSystem fs, Path path, CacheConfig cacheConfig, boolean primaryReplica) throws IOException { HStoreFile storeFile = null; try { storeFile = new HStoreFile(fs, path, conf, cacheConfig, BloomType.NONE, primaryReplica); storeFile.initReader(); } catch (IOException e) { LOG.error("Failed to open mob file[" + path + "], keep it in temp directory.", e); throw e; } finally { if (storeFile != null) { storeFile.closeStoreFile(false); } } }
for (HStoreFile file : files) { file.initReader(); sortedFiles.add(file);
/** * Validates a mob file by opening and closing it. * * @param path the path to the mob file */ private void validateMobFile(Path path) throws IOException { HStoreFile storeFile = null; try { storeFile = new HStoreFile(region.getFilesystem(), path, conf, this.cacheConf, BloomType.NONE, isPrimaryReplicaStore()); storeFile.initReader(); } catch (IOException e) { LOG.error("Fail to open mob file[" + path + "], keep it in temp directory.", e); throw e; } finally { if (storeFile != null) { storeFile.closeStoreFile(false); } } }
/** * Gets the max seqId and number of cells of the store files. * @param storeFiles The store files. * @return The pair of the max seqId and number of cells of the store files. * @throws IOException if IO failure is encountered */ private Pair<Long, Long> getFileInfo(List<HStoreFile> storeFiles) throws IOException { long maxSeqId = 0; long maxKeyCount = 0; for (HStoreFile sf : storeFiles) { // the readers will be closed later after the merge. maxSeqId = Math.max(maxSeqId, sf.getMaxSequenceId()); sf.initReader(); byte[] count = sf.getReader().loadFileInfo().get(MOB_CELLS_COUNT); if (count != null) { maxKeyCount += Bytes.toLong(count); } } return new Pair<>(maxSeqId, maxKeyCount); }
/** * Computes the length of a store file without succumbing to any errors along the way. If an * error is encountered, the implementation returns {@code 0} instead of the actual size. * * @param file The file to compute the size of. * @return The size in bytes of the provided {@code file}. */ long getStoreFileSize(HStoreFile file) { long length = 0; try { file.initReader(); length = file.getReader().length(); } catch (IOException e) { LOG.trace("Failed to open reader when trying to compute store file size, ignoring", e); } finally { try { file.closeStoreFile( file.getCacheConf() != null ? file.getCacheConf().shouldEvictOnClose() : true); } catch (IOException e) { LOG.trace("Failed to close reader after computing store file size, ignoring", e); } } return length; }
private HStoreFile createStoreFileAndReader(StoreFileInfo info) throws IOException { info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); HStoreFile storeFile = new HStoreFile(this.getFileSystem(), info, this.conf, this.cacheConf, this.family.getBloomFilterType(), isPrimaryReplicaStore()); storeFile.initReader(); return storeFile; }
private List<Cell> prepareListOfTestSeeks(Path path) throws IOException { List<Cell> allKeyValues = new ArrayList<>(); // read all of the key values HStoreFile storeFile = new HStoreFile(testingUtility.getTestFileSystem(), path, configuration, cacheConf, BloomType.NONE, true); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false, 0, 0, false); Cell current; scanner.seek(KeyValue.LOWESTKEY); while (null != (current = scanner.next())) { allKeyValues.add(current); } storeFile.closeStoreFile(cacheConf.shouldEvictOnClose()); // pick seeks by random List<Cell> seeks = new ArrayList<>(); for (int i = 0; i < numberOfSeeks; ++i) { Cell keyValue = allKeyValues.get( randomizer.nextInt(allKeyValues.size())); seeks.add(keyValue); } clearBlockCache(); return seeks; }
private long countMobCellsInMetadata() throws IOException { long mobCellsCount = 0; Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); if (fs.exists(mobDirPath)) { FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); for (FileStatus file : files) { HStoreFile sf = new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true); sf.initReader(); Map<byte[], byte[]> fileInfo = sf.getReader().loadFileInfo(); byte[] count = fileInfo.get(MOB_CELLS_COUNT); assertTrue(count != null); mobCellsCount += Bytes.toLong(count); } } return mobCellsCount; }
storeFile.initReader(); long totalSize = 0;
private boolean verifyEncryption(TableName tableName, String familyName) throws IOException { Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); boolean hasFiles = false; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); hasFiles = files != null && files.length > 0; Assert.assertTrue(hasFiles); Path path = files[0].getPath(); CacheConfig cacheConf = new CacheConfig(conf); HStoreFile sf = new HStoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf, BloomType.NONE, true); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); } return hasFiles; }
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
new HStoreFile(this.fs, storeFileInfo, testConf, cacheConf, BloomType.NONE, true); assertTrue(storeFileInfo.isLink()); hsf.initReader();
sf.initReader(); StoreFileReader reader = sf.getReader(); HFileScanner scanner = reader.getScanner(false, true);
storeFile.initReader(); StoreFileReader reader = storeFile.getReader();
BlockCache cache = cacheConf.getBlockCache().get(); HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.ROWCOL, true); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); try {
@Test public void testStoreFileReference() throws Exception { final RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf("testStoreFileReference")).build(); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(regionFs.createTempName()).withFileContext(meta).build(); writeStoreFile(writer); Path hsfPath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); writer.close(); HStoreFile file = new HStoreFile(this.fs, hsfPath, conf, cacheConf, BloomType.NONE, true); file.initReader(); StoreFileReader r = file.getReader(); assertNotNull(r); StoreFileScanner scanner = new StoreFileScanner(r, mock(HFileScanner.class), false, false, 0, 0, false); // Verify after instantiating scanner refCount is increased assertTrue("Verify file is being referenced", file.isReferencedInReads()); scanner.close(); // Verify after closing scanner refCount is decreased assertFalse("Verify file is not being referenced", file.isReferencedInReads()); }
FileSystem fs = FileSystem.get(conf); HStoreFile hsf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); StoreFileReader reader = hsf.getReader(); reader.loadFileInfo();
hsf.initReader(); StoreFileReader reader = hsf.getReader(); Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true); HStoreFile refHsf = new HStoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE, true); refHsf.initReader();