@Override public Collection<HStoreFile> getUnneededFiles(long maxTs, List<HStoreFile> filesCompacting) { ImmutableList<HStoreFile> files = storefiles; // 1) We can never get rid of the last file which has the maximum seqid. // 2) Files that are not the latest can't become one due to (1), so the rest are fair game. return files.stream().limit(Math.max(0, files.size() - 1)).filter(sf -> { long fileTs = sf.getReader().getMaxTimestamp(); if (fileTs < maxTs && !filesCompacting.contains(sf)) { LOG.info("Found an expired store file {} whose maxTimestamp is {}, which is below {}", sf.getPath(), fileTs, maxTs); return true; } else { return false; } }).collect(Collectors.toList()); }
private Collection<HStoreFile> findExpiredFiles(ImmutableList<HStoreFile> stripe, long maxTs, List<HStoreFile> filesCompacting, Collection<HStoreFile> expiredStoreFiles) { // Order by seqnum is reversed. for (int i = 1; i < stripe.size(); ++i) { HStoreFile sf = stripe.get(i); synchronized (sf) { long fileTs = sf.getReader().getMaxTimestamp(); if (fileTs < maxTs && !filesCompacting.contains(sf)) { LOG.info("Found an expired store file: " + sf.getPath() + " whose maxTimestamp is " + fileTs + ", which is below " + maxTs); if (expiredStoreFiles == null) { expiredStoreFiles = new ArrayList<>(); } expiredStoreFiles.add(sf); } } } return expiredStoreFiles; }
private Collection<HStoreFile> getExpiredStores(Collection<HStoreFile> files, Collection<HStoreFile> filesCompacting) { long currentTime = EnvironmentEdgeManager.currentTime(); Collection<HStoreFile> expiredStores = new ArrayList<>(); for (HStoreFile sf : files) { if (isEmptyStoreFile(sf)) { expiredStores.add(sf); continue; } // Check MIN_VERSIONS is in HStore removeUnneededFiles long maxTs = sf.getReader().getMaxTimestamp(); long maxTtl = storeConfigInfo.getStoreFileTtl(); if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)) { continue; } else if (filesCompacting == null || !filesCompacting.contains(sf)) { expiredStores.add(sf); } } return expiredStores; } }
private boolean hasExpiredStores(Collection<HStoreFile> files) { long currentTime = EnvironmentEdgeManager.currentTime(); for (HStoreFile sf : files) { if (isEmptyStoreFile(sf)) { return true; } // Check MIN_VERSIONS is in HStore removeUnneededFiles long maxTs = sf.getReader().getMaxTimestamp(); long maxTtl = storeConfigInfo.getStoreFileTtl(); if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)) { continue; } else { return true; } } return false; }
OUTER: for (int i = 0; i < stripes.size(); ++i) { for (HStoreFile storeFile : stripes.get(i)) { if (storeFile.getReader().getMaxTimestamp() < timestampCutoff) continue;
@SuppressWarnings("unchecked") @Test public void testMergeExpiredStripes() throws Exception { // HBASE-11397 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge all three expired stripes into one. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyMergeCompatcion(policy, si, 0, 2); // Merge two adjacent expired stripes into one. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyMergeCompatcion(policy, si, 3, 4); } finally { EnvironmentEdgeManager.reset(); } }
try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);
assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl)); assertEquals(1, sfs.size()); long ts = sfs.iterator().next().getReader().getMaxTimestamp(); assertTrue(ts < (edge.currentTime() - storeTtl));
long maxTimestampInFile = reader.getMaxTimestamp(); long seekTimestamp = kv.getTimestamp(); if (seekTimestamp > maxTimestampInFile) {
@SuppressWarnings("unchecked") @Test public void testMergeExpiredStripes() throws Exception { // HBASE-11397 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge all three expired stripes into one. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyMergeCompatcion(policy, si, 0, 2); // Merge two adjacent expired stripes into one. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyMergeCompatcion(policy, si, 3, 4); } finally { EnvironmentEdgeManager.reset(); } }
try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile);
assertTrue(sf.getReader().getMaxTimestamp() >= (edge.currentTime() - storeTtl)); assertEquals(1, sfs.size()); long ts = sfs.iterator().next().getReader().getMaxTimestamp(); assertTrue(ts < (edge.currentTime() - storeTtl));