@Override public void add(SortedOplog soplog) throws IOException { readers.addFirst(new TrackedReference<SortedOplogReader>(soplog.createReader())); }
/** * locks sorted oplogs collection and performs add operation * @return if addition was successful */ private boolean addSortedOplog(Hoplog so) throws IOException { if (logger.isDebugEnabled()) { logger.debug("{}Try add " + so, logPrefix); } hoplogRWLock.writeLock().lock(); try { int size = hoplogs.size(); boolean result = hoplogs.add(new TrackedReference<Hoplog>(so)); so.setReaderActivityListener(this); if (logger.isDebugEnabled()){ fineLog("Added: ", so, " Before:", size, " After:", hoplogs.size()); } return result; } finally { hoplogRWLock.writeLock().unlock(); } }
public void testGainComputeSmallFile() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB / 1024))); } float result = bucket.computeGain(2, 5, targets); assertTrue(Math.abs(8.0 - result) < (1.0/1000)); }
public void testGainComputeMixedFiles() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB / 1024))); } TestHoplog midHop = (TestHoplog) targets.get(4).get(); // one more than other files midHop.size = 5 * TEN_MB; float expect = (float) ((4 * 2 - 3 + 1.0) / 50); float result = bucket.computeGain(2, 5, targets); System.out.println(expect); System.out.println(result); assertTrue(Math.abs(expect - result) < (expect/1000)); }
/** * Tests hoplog file name comparator */ public void testHoplogFileComparator() throws IOException { String name1 = "bucket1-10-3.hop"; String name2 = "bucket1-1-20.hop"; String name3 = "bucket1-30-201.hop"; String name4 = "bucket1-100-201.hop"; TreeSet<TrackedReference<Hoplog>> list = new TreeSet<TrackedReference<Hoplog>>(new HoplogComparator()); // insert soplog is the list out of expected order hdfsStore.getFileSystem(); list.add(new TrackedReference<Hoplog>(new HFileSortedOplog(hdfsStore, new Path(testDataDir, name2), blockCache, stats, storeStats))); list.add(new TrackedReference<Hoplog>(new HFileSortedOplog(hdfsStore, new Path(testDataDir, name4), blockCache, stats, storeStats))); list.add(new TrackedReference<Hoplog>(new HFileSortedOplog(hdfsStore, new Path(testDataDir, name1), blockCache, stats, storeStats))); list.add(new TrackedReference<Hoplog>(new HFileSortedOplog(hdfsStore, new Path(testDataDir, name3), blockCache, stats, storeStats))); Iterator<TrackedReference<Hoplog>> iter = list.iterator(); assertEquals(name4, iter.next().get().getFileName()); assertEquals(name3, iter.next().get().getFileName()); assertEquals(name2, iter.next().get().getFileName()); assertEquals(name1, iter.next().get().getFileName()); }
public void testGainComputeBadRatio() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB))); } TestHoplog firstHop = (TestHoplog) targets.get(2).get(); // one more than other files firstHop.size = (1 + 30 + 40 + 50) * TEN_MB; Float result = bucket.computeGain(2, 5, targets); assertNull(result); }
public void testMinorCompactionTargetMinCount() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 2; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } compactor.getMinorCompactionTargets(targets, -1); assertEquals(0, targets.size()); }
public void testGainComputation() throws Exception { HoplogOrganizer<? extends PersistedEventImpl> organizer = regionManager.create(0); HdfsSortedOplogOrganizer bucket = (HdfsSortedOplogOrganizer) organizer; ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 10; i++) { targets.add(new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, i * TEN_MB))); } // each read has cost 3. Four files read cost is 3 * 4. Reduce read cost of // file after compaction float expect = (float) ((3 * 4.0 - 3) / (20 + 30 + 40 + 50)); float result = bucket.computeGain(2, 5, targets); assertTrue(Math.abs(expect - result) < (expect/1000)); // each read has cost 3 except 10MB file with read cost 2. 9 files read cost // is 3 * 9. Reduce read cost of file after compaction. expect = (float) ((3 * 9 - 3 - 1.0) / (10 + 20 + 30 + 40 + 50 + 60 + 70 + 80 + 90)); result = bucket.computeGain(0, 9, targets); assertTrue(Math.abs(expect - result) < (expect/1000)); }
/** * locks sorted oplogs collection and performs add operation * @return if addition was successful */ private boolean addSortedOplog(Hoplog so) throws IOException { if (logger.finerEnabled()) { logger.finer("Try add " + so); } hoplogRWLock.writeLock().lock(); try { int size = hoplogs.size(); boolean result = hoplogs.add(new TrackedReference<Hoplog>(so)); so.setReaderActivityListener(this); if (logger.fineEnabled()) { fineLog("Added: ", so, " Before:", size, " After:", hoplogs.size()); } return result; } finally { hoplogRWLock.writeLock().unlock(); } }
@Override protected void add(SortedOplog soplog) throws IOException { SortedOplogReader rdr = soplog.createReader(); synchronized (this) { soplogs.addFirst(new TrackedReference<SortedOplogReader>(rdr)); } if (logger.fineEnabled()) { logger.fine(String.format("Added file %s to level %d", rdr, level)); } tracker.fileAdded(rdr.getFile(), level); factory.getConfiguration().getStatistics().incActiveFiles(1); }
public void testTargetsReleasedBadRatio() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 3; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } TestHoplog oldestHop = (TestHoplog) targets.get(2).get(); oldestHop.size = (1 + 30) * TEN_MB; List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(0, list.size()); assertEquals(3, targets.size()); for (TrackedReference<Hoplog> ref : targets) { assertEquals(0, ref.uses()); } }
public void testMinorCompactionTargetsOptimizer() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); ArrayList<TrackedReference<Hoplog>> targets = new ArrayList<TrackedReference<Hoplog>>(); for (int i = 0; i < 6; i++) { TrackedReference<Hoplog> hop = new TrackedReference<Hoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(6, list.size()); TestHoplog fifthHop = (TestHoplog) targets.get(4).get(); // fifth hop needs additional block read as it has more than max keys size fifthHop.size = (HdfsSortedOplogOrganizer.AVG_NUM_KEYS_PER_INDEX_BLOCK * 5 + 1) * 64 * 1024; list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(4, list.size()); for (TrackedReference<Hoplog> ref : list) { assertTrue(((TestHoplog)ref.get()).size - TEN_MB < 4 ); } }
public void testAlterMaxInputFileSize() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); assertTrue(TEN_MB * 2 < hdfsStore.getHDFSCompactionConfig().getMaxInputFileSizeMB() * ONE_MB); ArrayList<TrackedReference<TestHoplog>> targets = new ArrayList<TrackedReference<TestHoplog>>(); for (int i = 0; i < 5; i++) { TrackedReference<TestHoplog> hop = new TrackedReference<TestHoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(targets.size(), list.size()); HDFSStoreMutator mutator = hdfsStore.createHdfsStoreMutator(); mutator.getCompactionConfigMutator().setMaxInputFileSizeMB(1); hdfsStore.alter(mutator); compactor.getMinorCompactionTargets(list, -1); assertEquals(0, list.size()); }
public void testAlterInputFileCount() throws Exception { HdfsSortedOplogOrganizer organizer = new HdfsSortedOplogOrganizer(regionManager, 0); HoplogCompactor compactor = (HoplogCompactor) organizer.getCompactor(); assertTrue(2 < hdfsStore.getHDFSCompactionConfig().getMaxInputFileCount()); ArrayList<TrackedReference<TestHoplog>> targets = new ArrayList<TrackedReference<TestHoplog>>(); for (int i = 0; i < 5; i++) { TrackedReference<TestHoplog> hop = new TrackedReference<TestHoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop); } List<TrackedReference<Hoplog>> list = (List<TrackedReference<Hoplog>>) targets.clone(); compactor.getMinorCompactionTargets(list, -1); assertEquals(targets.size(), list.size()); HDFSStoreMutator mutator = hdfsStore.createHdfsStoreMutator(); mutator.getCompactionConfigMutator().setMaxInputFileCount(2); mutator.getCompactionConfigMutator().setMinInputFileCount(2); hdfsStore.alter(mutator); compactor.getMinorCompactionTargets(list, -1); assertEquals(2, list.size()); }
TrackedReference<TestHoplog> hop = new TrackedReference<TestHoplog>(new TestHoplog(hdfsStore, TEN_MB + i)); hop.increment(); targets.add(hop);