@Override public void add(SortedOplog soplog) throws IOException { readers.addFirst(new TrackedReference<SortedOplogReader>(soplog.createReader())); }
private TrackedReference<Hoplog> trackHoplog(Hoplog hoplog, String user) { hoplogRWLock.readLock().lock(); try { for (TrackedReference<Hoplog> oplog : hoplogs) { if (oplog.get().getFileName().equals(hoplog.getFileName())) { oplog.increment(user); if (logger.isDebugEnabled()) { logger.debug("{}Track " + oplog, logPrefix); } return oplog; } } } finally { hoplogRWLock.readLock().unlock(); } throw new NoSuchElementException(hoplog.getFileName()); }
private void closeInactiveHoplogs() throws IOException { hoplogRWLock.writeLock().lock(); try { for (TrackedReference<Hoplog> hoplog : inactiveHoplogs) { if (logger.isDebugEnabled()){ logger.debug("{}Try close inactive " + hoplog, logPrefix); } if (!hoplog.inUse()) { int size = inactiveHoplogs.size(); inactiveHoplogs.remove(hoplog); closeReaderAndSuppressError(hoplog.get(), true); if (logger.isDebugEnabled()){ fineLog("Closed inactive: ", hoplog.get(), " Before:", size, " After:", inactiveHoplogs.size()); } } } } finally { hoplogRWLock.writeLock().unlock(); } }
public void testTargetsReleasedBadRatio() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 3; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } TestHoplog oldestHop = (TestHoplog) targets.get(2).get(); oldestHop.size = (1 + 30) * TEN_MB; List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(0, list.size()); assertEquals(3, targets.size()); for (TrackedReference<Hoplog> ref : targets) { assertEquals(0, ref.uses()); } }
public void testMinorCompactionTargetsOptimizer() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 6; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(6, list.size()); TestHoplog fifthHop = (TestHoplog) targets.get(4).get(); // fifth hop needs additional block read as it has more than max keys size fifthHop.size = (HdfsSortedOplogOrganizer.AVG_NUM_KEYS_PER_INDEX_BLOCK * 5 + 1) * 64 * 1024; list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(4, list.size()); for (TrackedReference<Hoplog> ref : list) { assertTrue(((TestHoplog)ref.get()).size - TEN_MB < 4 ); } }
public void testGainComputeBadRatio() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB))); } TestHoplog firstHop = (TestHoplog) targets.get(2).get(); // one more than other files firstHop.size = (1 + 30 + 40 + 50) * TEN_MB; Float result = bucket.computeGain(2, 5, targets); assertNull(result); }
target.decrement(user); if (logger.isDebugEnabled()) { logger.debug("{}Try release " + target, logPrefix); if (target.inUse()) { return; if (!target.inUse()) { if (inactiveHoplogs.contains(target) ) { int sizeBefore = inactiveHoplogs.size(); inactiveHoplogs.remove(target); closeReaderAndSuppressError(target.get(), true); if (logger.isDebugEnabled()) { fineLog("Closed inactive: ", target, " totalBefore:", sizeBefore, "Close reader: " + target.get().getFileName()), e); } finally { hoplogRWLock.writeLock().unlock();
public void testMinorCompactionTargetMinCount() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 2; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } compactor.getMinorCompactionTargets(targets, -1); assertEquals(0, targets.size()); }
/** * Increments the use counter and returns the current count. * @return the current uses */ public int increment() { return increment(null); }
soplogs = compactor.getActiveReaders(key, key); for (TrackedReference<SortedOplogReader> tr : soplogs) { readers.add(tr.get()); TrackedReference.decrementAll(soplogs);
/** * Returns true if the reference is in use. * @return true if used */ public boolean inUse() { return uses() > 0; }
/** * Decrements the use count of each reference. * @param refs the references to decrement */ public static <T> void decrementAll(Iterable<TrackedReference<T>> refs) { for (TrackedReference<?> tr : refs) { tr.decrement(); } }
protected void cleanupInactive() throws IOException { for (Iterator<TrackedReference<SortedOplogReader>> iter = inactive.iterator(); iter.hasNext(); ) { TrackedReference<SortedOplogReader> tr = iter.next(); if (!tr.inUse() && inactive.remove(tr)) { deleteInactive(tr); } } }
@Override public void close() { TrackedReference.decrementAll(soplogs); } }
TrackedReference<TestHoplog> hop = new TrackedReference<TestHoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); TestHoplog thirdHop = (TestHoplog) targets.get(2).get(); oldestHop.get().size = HDFSStore.HDFSCompactionConfig.DEFAULT_MAX_INPUT_FILE_SIZE_MB * ONE_MB + 100; List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(4, list.size()); for (TrackedReference<Hoplog> ref : list) { assertTrue(((TestHoplog)ref.get()).size - TEN_MB < 5 ); oldestHop.increment(); list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); for (TrackedReference<Hoplog> ref : list) { if (i != 2) { assertTrue(((TestHoplog) ref.get()).size - TEN_MB < 5); } else { assertTrue(((TestHoplog) ref.get()).size > HDFSStore.HDFSCompactionConfig.DEFAULT_MAX_INPUT_FILE_SIZE_MB * ONE_MB);
public void testGainComputeMixedFiles() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB / 1024))); } TestHoplog midHop = (TestHoplog) targets.get(4).get(); // one more than other files midHop.size = 5 * TEN_MB; float expect = (float) ((4 * 2 - 3 + 1.0) / 50); float result = bucket.computeGain(2, 5, targets); System.out.println(expect); System.out.println(result); assertTrue(Math.abs(expect - result) < (expect/1000)); }
private void closeExcessReaders() throws IOException { if (logger.isDebugEnabled()) { logger.debug("{}Close excess readers. Size:" + hoplogs.size() + " activeReaders:" + activeReaderCount.get() + " limit:" + maxOpenFilesLimit, logPrefix); } if (hoplogs.size() <= maxOpenFilesLimit) { return; } if (activeReaderCount.get() <= maxOpenFilesLimit) { return; } for (TrackedReference<Hoplog> hoplog : hoplogs.descendingSet()) { if (!hoplog.inUse() && !hoplog.get().isClosed()) { hoplog.get().close(false); if (logger.isDebugEnabled()) { logger.debug("{}Excess reader closed " + hoplog, logPrefix); } } if (activeReaderCount.get() <= maxOpenFilesLimit) { return; } } }
target.decrement(user); if (logger.finestEnabled()) { logger.finest("Try release " + target); if (target.inUse()) { return; if (!target.inUse()) { if (inactiveHoplogs.contains(target) ) { int sizeBefore = inactiveHoplogs.size(); inactiveHoplogs.remove(target); closeReaderAndSuppressError(target.get(), true); if (logger.fineEnabled()) { fineLog("Closed inactive: ", target, " totalBefore:", sizeBefore, "Close reader: " + target.get().getFileName(), e); } finally { hoplogRWLock.writeLock().unlock();