@Override public int getCompactPriority() { int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority(); if (priority == PRIORITY_USER) { LOG.warn("Compaction priority is USER despite there being no user compaction"); } return priority; }
/** * @return All store files. */ @Override public Collection<HStoreFile> getStorefiles() { return this.storeEngine.getStoreFileManager().getStorefiles(); }
@Override public int getStorefilesCount() { return this.storeEngine.getStoreFileManager().getStorefileCount(); }
@Override public long getNumReferenceFiles() { return this.storeEngine.getStoreFileManager().getStorefiles().stream() .filter(HStoreFile::isReference).count(); }
@Override public long getNumHFiles() { return this.storeEngine.getStoreFileManager().getStorefiles().stream() .filter(HStoreFile::isHFile).count(); }
@Override public Collection<HStoreFile> getCompactedFiles() { return this.storeEngine.getStoreFileManager().getCompactedfiles(); }
@Override public int getCompactedFilesCount() { return this.storeEngine.getStoreFileManager().getCompactedFilesCount(); }
@Override public double getCompactionPressure() { return storeEngine.getStoreFileManager().getCompactionPressure(); }
private long getStoreFileFieldSize(ToLongFunction<StoreFileReader> f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).map(HStoreFile::getReader).mapToLong(f).sum(); }
private void clearCompactedfiles(List<HStoreFile> filesToRemove) throws IOException { LOG.trace("Clearing the compacted file {} from this store", filesToRemove); try { lock.writeLock().lock(); this.getStoreEngine().getStoreFileManager().removeCompactedFiles(filesToRemove); } finally { lock.writeLock().unlock(); } }
/** Adds the files to compacting files. filesCompacting must be locked. */ private void addToCompactingFiles(Collection<HStoreFile> filesToAdd) { if (CollectionUtils.isEmpty(filesToAdd)) { return; } // Check that we do not try to compact the same StoreFile twice. if (!Collections.disjoint(filesCompacting, filesToAdd)) { Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting); } filesCompacting.addAll(filesToAdd); Collections.sort(filesCompacting, storeEngine.getStoreFileManager().getStoreFileComparator()); }
@Override public long getStorefilesSize() { // Include all StoreFiles return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), sf -> true); }
@Override public long getHFilesSize() { // Include only StoreFiles which are HFiles return getStorefilesSize(this.storeEngine.getStoreFileManager().getStorefiles(), HStoreFile::isHFile); }
@Override public boolean shouldPerformMajorCompaction() throws IOException { for (HStoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile {} has null Reader", sf); return false; } } return storeEngine.getCompactionPolicy().shouldPerformMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
/** * Determines if Store should be split. */ public Optional<byte[]> getSplitPoint() { this.lock.readLock().lock(); try { // Should already be enforced by the split policy! assert !this.getRegionInfo().isMetaRegion(); // Not split-able if we find a reference store file present in the store. if (hasReferences()) { LOG.trace("Not splittable; has references: {}", this); return Optional.empty(); } return this.storeEngine.getStoreFileManager().getSplitPoint(); } catch(IOException e) { LOG.warn("Failed getting store size for {}", this, e); } finally { this.lock.readLock().unlock(); } return Optional.empty(); }
private LongStream getStoreFileAgeStream() { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).filter(HStoreFile::isHFile).mapToLong(sf -> sf.getFileInfo().getCreatedTimestamp()) .map(t -> EnvironmentEdgeManager.currentTime() - t); }
@Override protected StoreEngine<?, ?, ?, ?> createStoreEngine(HStore store, Configuration conf, CellComparator kvComparator) throws IOException { List<HStoreFile> storefiles = Arrays.asList(mockStoreFile(currentTime - 10), mockStoreFile(currentTime - 100), mockStoreFile(currentTime - 1000), mockStoreFile(currentTime - 10000)); StoreFileManager sfm = mock(StoreFileManager.class); when(sfm.getStorefiles()).thenReturn(storefiles); StoreEngine<?, ?, ?, ?> storeEngine = mock(StoreEngine.class); when(storeEngine.getStoreFileManager()).thenReturn(sfm); return storeEngine; } };
private void closeCompactedFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStoreEngine().getStoreFileManager().getCompactedfiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } sf.closeStoreFile(true); store.getStoreEngine().getStoreFileManager().removeCompactedFiles(Lists.newArrayList(sf)); }
@Override public boolean evaluate() throws Exception { for (HRegion region : cluster.getRegions(tn)) { for (HStore store : region.getStores()) { Collection<HStoreFile> files = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (null != files && !files.isEmpty()) { LOG.debug(region.getRegionInfo().getEncodedName() + " still has compacted files"); return false; } } } return true; } }
private static List<Path> findCompactedStorefilePaths(TableName tableName) throws Exception { List<Path> paths = new ArrayList<>(); for (Region region : TEST_UTIL.getRSForFirstRegionInTable(tableName) .getRegions(tableName)) { for (HStore store : ((HRegion) region).getStores()) { Collection<HStoreFile> compactedfiles = store.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (compactedfiles != null) { for (HStoreFile storefile : compactedfiles) { paths.add(storefile.getPath()); } } } } return paths; }