public void dump() { System.out.println("map " + map.size()); System.out.println("isAllModified " + isAllModified()); System.out.println("isEmpty " + isEmpty()); for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) { entry.getValue().dump(); } }
private void removeInactiveLogs() throws IOException { // We keep track of which procedures are holding the oldest WAL in 'holdingCleanupTracker'. // once there is nothing olding the oldest WAL we can remove it. while (logs.size() > 1 && holdingCleanupTracker.isEmpty()) { LOG.info("Remove the oldest log {}", logs.getFirst()); removeLogFile(logs.getFirst(), walArchiveDir); buildHoldingCleanupTracker(); } // TODO: In case we are holding up a lot of logs for long time we should // rewrite old procedures (in theory parent procs) to the new WAL. }
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
private void periodicRoll() throws IOException { if (storeTracker.isEmpty()) { LOG.trace("no active procedures"); tryRollWriter(); removeAllLogs(flushLogId - 1, "no active procedures"); } else { if (storeTracker.isAllModified()) { LOG.trace("all the active procedures are in the latest log"); removeAllLogs(flushLogId - 1, "all the active procedures are in the latest log"); } // if the log size has exceeded the roll threshold // or the periodic roll timeout is expired, try to roll the wal. if (totalSynced.get() > rollThreshold || getMillisToNextPeriodicRoll() <= 0) { tryRollWriter(); } removeInactiveLogs(); } }
@Test public void testRandLoad() { final int NPROCEDURES = 2500; final int NRUNS = 5000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); Random rand = new Random(1); for (int i = 0; i < NRUNS; ++i) { assertTrue(tracker.isEmpty()); int count = 0; while (count < NPROCEDURES) { long procId = rand.nextLong(); if (procId < 1) continue; tracker.setDeleted(procId, i % 2 == 0); count++; } tracker.reset(); } }
@Test public void testSetDeletedIfModified() { final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); final long[] procIds = new long[] { 1, 3, 7, 152, 512, 1024, 1025 }; // test single proc for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); for (int i = 0; i < procIds.length; ++i) { tracker.setDeletedIfModified(procIds[i] - 1); tracker.setDeletedIfModified(procIds[i]); tracker.setDeletedIfModified(procIds[i] + 1); } assertEquals(true, tracker.isEmpty()); // test batch tracker.reset(); for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); tracker.setDeletedIfModified(procIds); assertEquals(true, tracker.isEmpty()); }
assertTrue(procStore.getStoreTracker().isEmpty()); assertEquals(1, procStore.getActiveLogs().size());
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }
/** * Tests that tracker for all old logs are loaded back after procedure store is restarted. */ @Test public void trackersLoadedForAllOldLogs() throws Exception { for (int i = 0; i <= 20; ++i) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 5) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); } } assertEquals(5, procStore.getActiveLogs().size()); for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); assertTrue(tracker != null && !tracker.isEmpty()); } }
@Test public void testBasicCRUD() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); assertTrue(tracker.isEmpty()); long[] procs = new long[] { 1, 2, 3, 4, 5, 6 }; tracker.insert(procs[0]); tracker.insert(procs[1], new long[] { procs[2], procs[3], procs[4] }); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); tracker.resetModified(); assertFalse(tracker.isAllModified()); for (int i = 0; i < 4; ++i) { tracker.update(procs[i]); assertFalse(tracker.isEmpty()); assertFalse(tracker.isAllModified()); } tracker.update(procs[4]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); tracker.update(procs[5]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); for (int i = 0; i < 5; ++i) { tracker.delete(procs[i]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); } tracker.delete(procs[5]); assertTrue(tracker.isEmpty()); }
@Test public void testPartialTracker() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); tracker.setPartialFlag(true); // nothing in the tracker, the state is unknown assertTrue(tracker.isEmpty()); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579)); // Mark 1 as deleted, now that is a known state tracker.setDeleted(1, true); tracker.dump(); assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579)); // Mark 579 as non-deleted, now that is a known state tracker.setDeleted(579, false); assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2)); assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(579)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(577)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(580)); tracker.setDeleted(579, true); tracker.setPartialFlag(false); assertTrue(tracker.isEmpty()); }
public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) { this.tracker = tracker; // we support fast-start only if we have a clean shutdown. this.hasFastStartSupport = !tracker.isEmpty(); }
public void dump() { System.out.println("map " + map.size()); System.out.println("isAllModified " + isAllModified()); System.out.println("isEmpty " + isEmpty()); for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) { entry.getValue().dump(); } }
public void dump() { System.out.println("map " + map.size()); System.out.println("isUpdated " + isUpdated()); System.out.println("isEmpty " + isEmpty()); for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) { entry.getValue().dump(); } }
public void dump() { System.out.println("map " + map.size()); System.out.println("isUpdated " + isUpdated()); System.out.println("isEmpty " + isEmpty()); for (Map.Entry<Long, BitSetNode> entry : map.entrySet()) { entry.getValue().dump(); } }
private void removeInactiveLogs() throws IOException { // We keep track of which procedures are holding the oldest WAL in 'holdingCleanupTracker'. // once there is nothing olding the oldest WAL we can remove it. while (logs.size() > 1 && holdingCleanupTracker.isEmpty()) { LOG.info("Remove the oldest log {}", logs.getFirst()); removeLogFile(logs.getFirst(), walArchiveDir); buildHoldingCleanupTracker(); } // TODO: In case we are holding up a lot of logs for long time we should // rewrite old procedures (in theory parent procs) to the new WAL. }
private void removeInactiveLogs() throws IOException { // We keep track of which procedures are holding the oldest WAL in 'holdingCleanupTracker'. // once there is nothing olding the oldest WAL we can remove it. while (logs.size() > 1 && holdingCleanupTracker.isEmpty()) { removeLogFile(logs.getFirst(), walArchiveDir); buildHoldingCleanupTracker(); } // TODO: In case we are holding up a lot of logs for long time we should // rewrite old procedures (in theory parent procs) to the new WAL. }
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }