private LongStream getStoreFileAgeStream() { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).filter(HStoreFile::isHFile).mapToLong(sf -> sf.getFileInfo().getCreatedTimestamp()) .map(t -> EnvironmentEdgeManager.currentTime() - t); }
currentFilesSet.put(sf.getFileInfo(), sf); compactedFilesSet.put(sf.getFileInfo(), sf);
private HStoreFile mockStoreFile(long createdTime) { StoreFileInfo info = mock(StoreFileInfo.class); when(info.getCreatedTimestamp()).thenReturn(createdTime); HStoreFile sf = mock(HStoreFile.class); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.isHFile()).thenReturn(true); when(sf.getFileInfo()).thenReturn(info); return sf; }
private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); // Append the new storefile into the list this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); } finally { // We need the lock, as long as we are updating the storeFiles // or changing the memstore. Let us release it before calling // notifyChangeReadersObservers. See HBASE-4485 for a possible // deadlock scenario that could have happened if continue to hold // the lock. this.lock.writeLock().unlock(); } LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName()); if (LOG.isTraceEnabled()) { String traceMessage = "BULK LOAD time,size,store size,store files [" + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } }
new HashMap<>(getStorefilesCount() + getCompactedFilesCount()); for (HStoreFile file : getStorefiles()) { name2File.put(file.getFileInfo().getActiveFileName(), file); name2File.put(file.getFileInfo().getActiveFileName(), file);
@VisibleForTesting protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException { // 1. dump region meta info into the snapshot directory LOG.debug("Storing '" + region + "' region-info for snapshot."); Object regionData = visitor.regionOpen(region.getRegionInfo()); monitor.rethrowException(); // 2. iterate through all the stores in the region LOG.debug("Creating references for hfiles"); for (HStore store : region.getStores()) { // 2.1. build the snapshot reference for the store Object familyData = visitor.familyOpen(regionData, store.getColumnFamilyDescriptor().getName()); monitor.rethrowException(); List<HStoreFile> storeFiles = new ArrayList<>(store.getStorefiles()); if (LOG.isDebugEnabled()) { LOG.debug("Adding snapshot references for " + storeFiles + " hfiles"); } // 2.2. iterate through all the store's files and create "references". for (int i = 0, sz = storeFiles.size(); i < sz; i++) { HStoreFile storeFile = storeFiles.get(i); monitor.rethrowException(); // create "reference" to this store file. LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath()); visitor.storeFile(regionData, familyData, storeFile.getFileInfo()); } visitor.familyClose(regionData, familyData); } visitor.regionClose(regionData); }
assertTrue("Column family " + fn + " should have 3 copies", FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf .getFileInfo().getFileStatus().getReplication())); for (HStoreFile sf : store.getStorefiles()) { assertTrue(sf.toString().contains(fn1)); assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo() .getFileStatus().getReplication());
private HStoreFile mockStoreFile(long createdTime) { StoreFileInfo info = mock(StoreFileInfo.class); when(info.getCreatedTimestamp()).thenReturn(createdTime); HStoreFile sf = mock(HStoreFile.class); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.isHFile()).thenReturn(true); when(sf.getFileInfo()).thenReturn(info); return sf; }
assertTrue("Column family " + fn + " should have 3 copies", FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf .getFileInfo().getFileStatus().getReplication())); for (HStoreFile sf : store.getStorefiles()) { assertTrue(sf.toString().contains(fn1)); assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo() .getFileStatus().getReplication());