@Override public synchronized void clear() throws IOException { for (TrackedReference<SortedOplogReader> tr : readers) { tr.get().close(); readers.remove(tr); } }
private boolean readerIsClosed() { for (TrackedReference<SortedOplogReader> tr : soplogs) { if (tr.get().isClosed()) { return true; } } return false; } }
@Override protected synchronized void clear() throws IOException { for (TrackedReference<SortedOplogReader> tr : soplogs) { tr.get().close(); } markAsInactive(soplogs, level); soplogs.clear(); }
/** * locks sorted oplogs collection, removes oplog and renames for deletion later * @throws IOException */ void markSortedOplogForDeletion(List<TrackedReference<Hoplog>> targets, boolean notify) throws IOException { for (int i = targets.size(); i > 0; i--) { TrackedReference<Hoplog> so = targets.get(i - 1); removeSortedOplog(so, true); if (!store.getFileSystem().exists(new Path(bucketPath, so.get().getFileName()))) { // the hoplog does not even exist on file system. Skip remaining steps continue; } addExpiryMarkerForAFile(so.get()); } }
/** * locks sorted oplogs collection, removes oplog and renames for deletion later * @throws IOException */ void markSortedOplogForDeletion(List<TrackedReference<Hoplog>> targets, boolean notify) throws IOException { for (int i = targets.size(); i > 0; i--) { TrackedReference<Hoplog> so = targets.get(i - 1); removeSortedOplog(so, true); if (!store.getFileSystem().exists(new Path(bucketPath, so.get().getFileName()))) { // the hoplog does not even exist on file system. Skip remaining steps continue; } addExpiryMarkerForAFile(so.get()); } }
private void removeSortedOplog(TrackedReference<Hoplog> so, boolean notify) throws IOException { hoplogReadersController.removeSortedOplog(so); // release lock before notifying listeners if (notify && listener != null) { listener.hoplogDeleted(regionFolder, bucketId, so.get()); } }
private void removeSortedOplog(TrackedReference<Hoplog> so, boolean notify) throws IOException { hoplogReadersController.removeSortedOplog(so); // release lock before notifying listeners if (notify && listener != null) { listener.hoplogDeleted(regionFolder, bucketId, so.get()); } }
Hoplog getOldestHoplog() { if (hoplogs.isEmpty()) { return null; } return hoplogs.last().get(); }
Hoplog getOldestHoplog() { if (hoplogs.isEmpty()) { return null; } return hoplogs.last().get(); }
private void deleteInactive(TrackedReference<SortedOplogReader> tr) throws IOException { tr.get().close(); if (tr.get().getFile().delete()) { if (logger.fineEnabled()) { logger.fine("Deleted inactive soplog " + tr.get().getFile()); } tracker.fileDeleted(tr.get().getFile()); factory.getConfiguration().getStatistics().incInactiveFiles(-1); } }
public HoplogSetIterator(List<TrackedReference<Hoplog>> targets) throws IOException { iters = new ArrayList<HFileSortedIterator>(); for (TrackedReference<Hoplog> oplog : targets) { HFileSortedIterator iter = (HFileSortedIterator) oplog.get().getReader().scan(); if (!iter.hasNext()) { // the oplog is empty, exclude from iterator continue; } // initialize the iterator iter.nextBB(); iters.add(iter); entriesRemaining += oplog.get().getReader().getEntryCount(); } }
public HoplogSetIterator(List<TrackedReference<Hoplog>> targets) throws IOException { iters = new ArrayList<HFileSortedIterator>(); for (TrackedReference<Hoplog> oplog : targets) { HFileSortedIterator iter = (HFileSortedIterator) oplog.get().getReader().scan(); if (!iter.hasNext()) { // the oplog is empty, exclude from iterator continue; } // initialize the iterator iter.nextBB(); iters.add(iter); entriesRemaining += oplog.get().getReader().getEntryCount(); } }
@Override protected synchronized void close() throws IOException { for (TrackedReference<SortedOplogReader> tr : soplogs) { tr.get().close(); factory.getConfiguration().getStatistics().incActiveFiles(-1); } soplogs.clear(); }
public void testGainComputeBadRatio() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB))); } TestHoplog firstHop = (TestHoplog) targets.get(2).get(); // one more than other files firstHop.size = (1 + 30 + 40 + 50) * TEN_MB; Float result = bucket.computeGain(2, 5, targets); assertNull(result); }
protected void markAsInactive(Iterable<TrackedReference<SortedOplogReader>> snapshot, T attach) throws IOException { for (Iterator<TrackedReference<SortedOplogReader>> iter = snapshot.iterator(); iter.hasNext(); ) { TrackedReference<SortedOplogReader> tr = iter.next(); if (logger.fineEnabled()) { logger.fine(String.format("Marking %s as inactive", tr)); } inactive.add(tr); tracker.fileRemoved(tr.get().getFile(), attach); factory.getConfiguration().getStatistics().incActiveFiles(-1); factory.getConfiguration().getStatistics().incInactiveFiles(1); } }
public void testForceCompaction() throws Exception { compactor.add(createSoplog(0, 100, 0)); compactor.add(createSoplog(100, 100, 1)); boolean compacted = compactor.compact(); assertTrue(compacted); validate(compactor.getActiveReaders(null, null).iterator().next().get(), 0, 200); }
public void testMultilevelCompaction() throws Exception { for (int i = 0; i < 8; i += 2) { compactor.add(createSoplog(0, 100, i)); compactor.add(createSoplog(100, 100, i+1)); WaitingHandler wh = new WaitingHandler(); compactor.compact(false, wh); wh.waitForCompletion(); } assertEquals(1, compactor.getActiveReaders(null, null).size()); validate(compactor.getActiveReaders(null, null).iterator().next().get(), 0, 200); }
private void reEstimateBucketSize() throws IOException { ICardinality global = null; String user = logger.fineEnabled() ? "HLL" : null; List<TrackedReference<Hoplog>> hoplogs = null; try { hoplogs = hoplogReadersController.getTrackedSortedOplogList(user); global = new HyperLogLog(HLL_CONSTANT); for (TrackedReference<Hoplog> hop : hoplogs) { global = mergeHLL(global, hop.get().getEntryCountEstimate()); } } finally { hoplogReadersController.releaseHoplogs(hoplogs, user); } bucketSize = global; }