/** * Gets the largest file (with reader) out of the list of files. * @param candidates The files to choose from. * @return The largest file; null if no file has a reader. */ static Optional<HStoreFile> getLargestFile(Collection<HStoreFile> candidates) { return candidates.stream().filter(f -> f.getReader() != null) .max((f1, f2) -> Long.compare(f1.getReader().length(), f2.getReader().length())); }
/** * Get a scanner which uses pread. * <p> * Must be called after initReader. */ public StoreFileScanner getPreadScanner(boolean cacheBlocks, long readPt, long scannerOrder, boolean canOptimizeForNonNullColumn) { return getReader().getStoreFileScanner(cacheBlocks, true, false, readPt, scannerOrder, canOptimizeForNonNullColumn); }
/** * Find the total size of a list of store files. * @param potentialMatchFiles StoreFile list. * @return Sum of StoreFile.getReader().length(); */ private long getTotalStoreSize(List<HStoreFile> potentialMatchFiles) { return potentialMatchFiles.stream().mapToLong(sf -> sf.getReader().length()).sum(); }
public static long getTotalFileSize(final Collection<HStoreFile> candidates) { long totalSize = 0; for (HStoreFile storeFile : candidates) { totalSize += storeFile.getReader().length(); } return totalSize; }
/** * The FIFOCompactionPolicy only choose those TTL expired HFiles as the compaction candidates. So * if all HFiles are TTL expired, then the compaction will generate a new empty HFile. While its * max timestamp will be Long.MAX_VALUE. If not considered separately, the HFile will never be * archived because its TTL will be never expired. So we'll check the empty store file separately. * (See HBASE-21504) */ private boolean isEmptyStoreFile(HStoreFile sf) { return sf.getReader().getEntries() == 0; }
/** * Gets the total size of all files in the stripe. * @param stripeIndex Stripe index. * @return Size. */ private long getStripeFilesSize(int stripeIndex) { long result = 0; for (HStoreFile sf : state.stripeFiles.get(stripeIndex)) { result += sf.getReader().length(); } return result; }
private static long getTotalKvCount(final Collection<HStoreFile> candidates) { long totalSize = 0; for (HStoreFile storeFile : candidates) { totalSize += storeFile.getReader().getEntries(); } return totalSize; }
private long getStoreFileFieldSize(ToLongFunction<StoreFileReader> f) { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).map(HStoreFile::getReader).mapToLong(f).sum(); }
@Override public String toString() { String fsList = filesToCompact.stream().filter(f -> f.getReader() != null) .map(f -> TraditionalBinaryPrefix.long2String(f.getReader().length(), "", 1)) .collect(Collectors.joining(", ")); return "regionName=" + regionName + ", storeName=" + storeName + ", fileCount=" + this.getFiles().size() + ", fileSize=" + TraditionalBinaryPrefix.long2String(totalSize, "", 1) + ((fsList.isEmpty()) ? "" : " (" + fsList + ")") + ", priority=" + priority + ", time=" + selectionTime; }
@Override public OptionalLong getMaximumTimestamp() { TimeRange tr = getReader().timeRange; return tr != null ? OptionalLong.of(tr.getMax()) : OptionalLong.empty(); }
@Override public OptionalLong getMinimumTimestamp() { TimeRange tr = getReader().timeRange; return tr != null ? OptionalLong.of(tr.getMin()) : OptionalLong.empty(); }
private LongStream getStoreFileAgeStream() { return this.storeEngine.getStoreFileManager().getStorefiles().stream().filter(sf -> { if (sf.getReader() == null) { LOG.warn("StoreFile {} has a null Reader", sf); return false; } else { return true; } }).filter(HStoreFile::isHFile).mapToLong(sf -> sf.getFileInfo().getCreatedTimestamp()) .map(t -> EnvironmentEdgeManager.currentTime() - t); }
long[] getSizes(List<HStoreFile> sfList) { long[] aNums = new long[sfList.size()]; for (int i = 0; i < sfList.size(); ++i) { aNums[i] = sfList.get(i).getReader().length(); } return aNums; }
private HStoreFile mockStoreFile(long createdTime) { StoreFileInfo info = mock(StoreFileInfo.class); when(info.getCreatedTimestamp()).thenReturn(createdTime); HStoreFile sf = mock(HStoreFile.class); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.isHFile()).thenReturn(true); when(sf.getFileInfo()).thenReturn(info); return sf; }
private static HStoreFile createFile() throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("file")); StoreFileReader r = mock(StoreFileReader.class); when(r.length()).thenReturn(10L); when(sf.getReader()).thenReturn(r); return sf; }
private HStoreFile mockStoreFileWithLength(long length) { HStoreFile sf = mock(HStoreFile.class); StoreFileReader sfr = mock(StoreFileReader.class); when(sf.isHFile()).thenReturn(true); when(sf.getReader()).thenReturn(sfr); when(sfr.length()).thenReturn(length); return sf; }
private static HStoreFile createFile() throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getMetadataValue(any())) .thenReturn(StripeStoreFileManager.INVALID_KEY); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.getPath()).thenReturn(new Path("moo")); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
@Override public boolean shouldPerformMajorCompaction() throws IOException { for (HStoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) { // TODO: what are these reader checks all over the place? if (sf.getReader() == null) { LOG.debug("StoreFile {} has null Reader", sf); return false; } } return storeEngine.getCompactionPolicy().shouldPerformMajorCompaction( this.storeEngine.getStoreFileManager().getStorefiles()); }
private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) { HStoreFile storeFile = storefiles.iterator().next(); HFile.Reader reader = storeFile.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); }