/** * @return All store files. */ @Override public Collection<HStoreFile> getStorefiles() { return this.storeEngine.getStoreFileManager().getStorefiles(); }
@Override public long getNumReferenceFiles() { return this.storeEngine.getStoreFileManager().getStorefiles().stream() .filter(HStoreFile::isReference).count(); }
@Override public long getNumHFiles() { return this.storeEngine.getStoreFileManager().getStorefiles().stream() .filter(HStoreFile::isHFile).count(); }
private long getStoreFileFieldSize(ToLongFunction<StoreFileReader> f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).map(HStoreFile::getReader).mapToLong(f).sum(); }
@Override public long getStorefilesSize() { // Include all StoreFiles return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), sf -> true); }
@Override public long getHFilesSize() { // Include only StoreFiles which are HFiles return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), HStoreFile::isHFile); }
@Override public boolean shouldPerformMajorCompaction() throws IOException { for (HStoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile {} has null Reader", sf); return false; } } return storeEngine.getCompactionPolicy().shouldPerformMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
private LongStream getStoreFileAgeStream() { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).filter(HStoreFile::isHFile).mapToLong(sf -> sf.getFileInfo().getCreatedTimestamp()) .map(t -> EnvironmentEdgeManager.currentTime() - t); }
@Override protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf, CellComparator kvComparator) throws IOException { List<HStoreFile> storefiles = Arrays.asList(mockStoreFile(currentTime - 10), mockStoreFile(currentTime - 100), mockStoreFile(currentTime - 1000), mockStoreFile(currentTime - 10000)); StoreFileManager sfm = mock(StoreFileManager.class); when(sfm.getStorefiles()).thenReturn(storefiles); StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class); when(storeEngine.getStoreFileManager()).thenReturn(sfm); return storeEngine; } };
/** * <p>It works by processing a compaction that's been written to disk. * * <p>It is usually invoked at the end of a compaction, but might also be * invoked at HStore startup, if the prior execution died midway through. * * <p>Moving the compacted TreeMap into place means: * <pre> * 1) Unload all replaced StoreFile, close and collect list to delete. * 2) Compute new store size * </pre> * * @param compactedFiles list of files that were compacted */ @VisibleForTesting protected void completeCompaction(Collection<HStoreFile> compactedFiles) throws IOException { this.storeSize.set(0L); this.totalUncompressedBytes.set(0L); for (HStoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFileReader r = hsf.getReader(); if (r == null) { LOG.warn("StoreFile {} has a null Reader", hsf); continue; } this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); } }
Collection<HStoreFile> currentFiles = sfm.getStorefiles(); Collection<HStoreFile> compactedFiles = sfm.getCompactedfiles(); if (currentFiles == null) currentFiles = Collections.emptySet();
try { synchronized (filesCompacting) { filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles()); if (!filesCompacting.isEmpty()) {
/** * @return All store files. */ @Override public Collection<StoreFile> getStorefiles() { return this.storeEngine.getStoreFileManager().getStorefiles(); }
@Override public boolean hasReferences() { return StoreUtils.hasReferences(this.storeEngine.getStoreFileManager().getStorefiles()); }
@Override protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf, CellComparator kvComparator) throws IOException { List<HStoreFile> storefiles = Arrays.asList(mockStoreFile(currentTime - 10), mockStoreFile(currentTime - 100), mockStoreFile(currentTime - 1000), mockStoreFile(currentTime - 10000)); StoreFileManager sfm = mock(StoreFileManager.class); when(sfm.getStorefiles()).thenReturn(storefiles); StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class); when(storeEngine.getStoreFileManager()).thenReturn(sfm); return storeEngine; } };
@Override public long getTotalStaticBloomSize() { long size = 0; for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFile.Reader r = s.getReader(); if (r == null) { continue; } size += r.getTotalBloomSize(); } return size; }
@Override public boolean isMajorCompaction() throws IOException { for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile " + sf + " has null Reader"); return false; } } return storeEngine.getCompactionPolicy().isMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
@Override public long getTotalStaticIndexSize() { long size = 0; for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFile.Reader r = s.getReader(); if (r == null) { continue; } size += r.getUncompressedDataIndexSize(); } return size; }
@Override public long getStorefilesSize() { long size = 0; for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFile.Reader r = s.getReader(); if (r == null) { LOG.warn("StoreFile " + s + " has a null Reader"); continue; } size += r.length(); } return size; }
@Override public long getStorefilesIndexSize() { long size = 0; for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) { StoreFile.Reader r = s.getReader(); if (r == null) { LOG.warn("StoreFile " + s + " has a null Reader"); continue; } size += r.indexSize(); } return size; }