@VisibleForTesting void replaceStoreFiles(Collection<HStoreFile> compactedFiles, Collection<HStoreFile> result) throws IOException { this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result); synchronized (filesCompacting) { filesCompacting.removeAll(compactedFiles); } // These may be null when the RS is shutting down. The space quota Chores will fix the Region // sizes later so it's not super-critical if we miss these. RegionServerServices rsServices = region.getRegionServerServices(); if (rsServices != null && rsServices.getRegionServerSpaceQuotaManager() != null) { updateSpaceQuotaAfterFileReplacement( rsServices.getRegionServerSpaceQuotaManager().getRegionSizeStore(), getRegionInfo(), compactedFiles, result); } } finally { this.lock.writeLock().unlock(); } }
/** * @return All store files. */ @Override public Collection<HStoreFile> getStorefiles() { return this.storeEngine.getStoreFileManager().getStorefiles(); }
try { ImmutableCollection<HStoreFile> result = storeEngine.getStoreFileManager().clearFiles(); Collection<HStoreFile> compactedfiles = storeEngine.getStoreFileManager().clearCompactedFiles();
private void bulkLoadHFile(HStoreFile sf) throws IOException { StoreFileReader r = sf.getReader(); this.storeSize.addAndGet(r.length()); this.totalUncompressedBytes.addAndGet(r.getTotalUncompressedBytes()); // Append the new storefile into the list this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().insertNewFiles(Lists.newArrayList(sf)); } finally { // We need the lock, as long as we are updating the storeFiles // or changing the memstore. Let us release it before calling // notifyChangeReadersObservers. See HBASE-4485 for a possible // deadlock scenario that could have happened if continue to hold // the lock. this.lock.writeLock().unlock(); } LOG.info("Loaded HFile " + sf.getFileInfo() + " into store '" + getColumnFamilyName()); if (LOG.isTraceEnabled()) { String traceMessage = "BULK LOAD time,size,store size,store files [" + EnvironmentEdgeManager.currentTime() + "," + r.length() + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage); } }
try { synchronized (filesCompacting) { filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles()); if (!filesCompacting.isEmpty()) { isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); Collections.sort(filesCompacting, storeEngine.getStoreFileManager() .getStoreFileComparator());
Collection<HStoreFile> currentFiles = sfm.getStorefiles(); Collection<HStoreFile> compactedFiles = sfm.getCompactedfiles(); if (currentFiles == null) currentFiles = Collections.emptySet(); if (newFiles == null) newFiles = Collections.emptySet();
@Override public Collection<HStoreFile> getCompactedFiles() { return this.storeEngine.getStoreFileManager().getCompactedfiles(); }
Collection<HStoreFile> initialCompactedFiles = fileManager.getCompactedfiles(); assertTrue(initialCompactedFiles == null || initialCompactedFiles.isEmpty()); Collection<HStoreFile> compactedFilesPreClean = fileManager.getCompactedfiles(); assertNotNull(compactedFilesPreClean); assertTrue(compactedFilesPreClean.size() > 0); fileManager.addCompactionResults( ImmutableList.of(errStoreFile), ImmutableList.of()); Collection<HStoreFile> compactedFilesPostClean = fileManager.getCompactedfiles(); assertEquals(1, compactedFilesPostClean.size()); for (HStoreFile origFile : compactedFilesPreClean) {
this.storeEngine.getStoreFileManager().loadFiles(hStoreFiles);
private void closeCompactedFile(int index) throws IOException { Collection<HStoreFile> files = this.store.getStoreEngine().getStoreFileManager().getCompactedfiles(); HStoreFile sf = null; Iterator<HStoreFile> it = files.iterator(); for (int i = 0; i <= index; i++) { sf = it.next(); } sf.closeStoreFile(true); store.getStoreEngine().getStoreFileManager().removeCompactedFiles(Lists.newArrayList(sf)); }
try { synchronized (filesCompacting) { filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles()); if (!filesCompacting.isEmpty()) { isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
@Override public int getStorefilesCount() { return this.storeEngine.getStoreFileManager().getStorefileCount(); }
@Override public int getCompactPriority() { int priority = this.storeEngine.getStoreFileManager().getStoreCompactionPriority(); if (priority == PRIORITY_USER) { LOG.warn("Compaction priority is USER despite there being no user compaction"); } return priority; }
@Override public double getCompactionPressure() { return storeEngine.getStoreFileManager().getCompactionPressure(); }
/** * Determines if Store should be split. */ public Optional<byte[]> getSplitPoint() { this.lock.readLock().lock(); try { // Should already be enforced by the split policy! assert !this.getRegionInfo().isMetaRegion(); // Not split-able if we find a reference store file present in the store. if (hasReferences()) { LOG.trace("Not splittable; has references: {}", this); return Optional.empty(); } return this.storeEngine.getStoreFileManager().getSplitPoint(); } catch(IOException e) { LOG.warn("Failed getting store size for {}", this, e); } finally { this.lock.readLock().unlock(); } return Optional.empty(); }
long cfTtl = getStoreFileTtl(); if (cfTtl != Long.MAX_VALUE) { delSfs = storeEngine.getStoreFileManager().getUnneededFiles( EnvironmentEdgeManager.currentTime() - cfTtl, filesCompacting); addToCompactingFiles(delSfs);
try { ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();
this.lock.writeLock().lock(); try { this.storeEngine.getStoreFileManager().insertNewFiles(sfs); if (snapshotId > 0) { this.memstore.clearSnapshot(snapshotId); String traceMessage = "FLUSH time,count,size,store size,store files [" + EnvironmentEdgeManager.currentTime() + "," + sfs.size() + "," + totalSize + "," + storeSize + "," + storeEngine.getStoreFileManager().getStorefileCount() + "]"; LOG.trace(traceMessage);
@VisibleForTesting public synchronized void closeAndArchiveCompactedFiles(boolean storeClosing) throws IOException { // ensure other threads do not attempt to archive the same files on close() archiveLock.lock(); try { lock.readLock().lock(); Collection<HStoreFile> copyCompactedfiles = null; try { Collection<HStoreFile> compactedfiles = this.getStoreEngine().getStoreFileManager().getCompactedfiles(); if (CollectionUtils.isNotEmpty(compactedfiles)) { // Do a copy under read lock copyCompactedfiles = new ArrayList<>(compactedfiles); } else { LOG.trace("No compacted files to archive"); } } finally { lock.readLock().unlock(); } if (CollectionUtils.isNotEmpty(copyCompactedfiles)) { removeCompactedfiles(copyCompactedfiles, storeClosing); } } finally { archiveLock.unlock(); } }
Collection<HStoreFile> initialCompactedFiles = fileManager.getCompactedfiles(); assertTrue(initialCompactedFiles == null || initialCompactedFiles.isEmpty()); Collection<HStoreFile> compactedFilesPreClean = fileManager.getCompactedfiles(); assertNotNull(compactedFilesPreClean); assertTrue(compactedFilesPreClean.size() > 0); fileManager.addCompactionResults( ImmutableList.of(errStoreFile), ImmutableList.of()); Collection<HStoreFile> compactedFilesPostClean = fileManager.getCompactedfiles(); assertEquals(1, compactedFilesPostClean.size()); for (HStoreFile origFile : compactedFilesPreClean) {