public int getNumWALFiles() { return procedureStore != null ? procedureStore.getActiveLogs().size() : 0; }
private void tryCleanupLogsOnLoad() { // nothing to cleanup. if (logs.size() <= 1) { return; } // the config says to not cleanup wals on load. if (!conf.getBoolean(EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, DEFAULT_EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY)) { LOG.debug("WALs cleanup on load is not enabled: " + getActiveLogs()); return; } try { periodicRoll(); } catch (IOException e) { LOG.warn("Unable to cleanup logs on load: " + e.getMessage(), e); } }
@Test public void testRestartWithoutData() throws Exception { for (int i = 0; i < 10; ++i) { final LoadCounter loader = new LoadCounter(); storeRestart(loader); } LOG.info("ACTIVE WALs " + procStore.getActiveLogs()); assertEquals(1, procStore.getActiveLogs().size()); FileStatus[] status = fs.listStatus(logDir); assertEquals(1, status.length); }
private void setupProcedureStore() throws IOException { Path testDir = UTIL.getDataTestDir(); FileSystem fs = testDir.getFileSystem(conf); Path logDir = new Path(testDir, "proc-logs"); System.out.println("Logs directory : " + logDir.toString()); fs.delete(logDir, true); if ("nosync".equals(syncType)) { store = new NoSyncWalProcedureStore(conf, logDir); } else { store = ProcedureTestingUtility.createWalStore(conf, logDir); } store.start(numThreads); store.recoverLease(); store.load(new ProcedureTestingUtility.LoadCounter()); System.out.println("Starting new log : " + store.getActiveLogs().get(store.getActiveLogs().size() - 1)); }
@Test public void testEmptyRoll() throws Exception { for (int i = 0; i < 10; ++i) { procStore.periodicRollForTesting(); } assertEquals(1, procStore.getActiveLogs().size()); FileStatus[] status = fs.listStatus(logDir); assertEquals(1, status.length); }
/** * Tests that tracker for all old logs are loaded back after procedure store is restarted. */ @Test public void trackersLoadedForAllOldLogs() throws Exception { for (int i = 0; i <= 20; ++i) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 5) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); } } assertEquals(5, procStore.getActiveLogs().size()); for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); assertTrue(tracker != null && !tracker.isEmpty()); } }
store.rollWriterForTesting(); System.out.println("Starting new log : " + store.getActiveLogs().get(store.getActiveLogs().size() - 1));
@Test public void testWalCleanerWithEmptyRolls() throws Exception { final Procedure<?>[] procs = new Procedure[3]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); } assertEquals(1, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(3, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting(); procStore.rollWriterForTesting(); if (i < (procs.length - 1)) { assertEquals(3 + ((i + 1) * 2), procStore.getActiveLogs().size()); } } assertEquals(7, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.delete(procs[i].getProcId()); assertEquals(7 - ((i + 1) * 2), procStore.getActiveLogs().size()); } assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerSequentialClean() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(logs.size(), i + 2); // Extra 1 for current ongoing wal. } // Delete procedures in sequential order make sure that only the corresponding wal is deleted // from logs list. final int[] deleteOrder = new int[] { 0, 1, 2, 3, 4 }; for (int i = 0; i < deleteOrder.length; i++) { procStore.delete(procs[deleteOrder[i]].getProcId()); procStore.removeInactiveLogsForTesting(); assertFalse(logs.get(deleteOrder[i]).toString(), procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); assertEquals(procStore.getActiveLogs().size(), procs.length - i); } }
@Test public void testProcedureUpdatedShouldClean() throws Exception { createProcExecutor(); SuspendProcedure suspendProcedure = new SuspendProcedure(); long suspendProc = procExecutor.submitProcedure(suspendProcedure); LOG.info("Begin to execute " + suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(suspendProc) != null); // Wait until the suspendProc executed after restart suspendProcedure = (SuspendProcedure) procExecutor.getProcedure(suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@Test public void testProcedureDeletedShouldClean() throws Exception { createProcExecutor(); WaitProcedure waitProcedure = new WaitProcedure(); long waitProce = procExecutor.submitProcedure(waitProcedure); LOG.info("Begin to execute " + waitProce); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(waitProce) != null); // Wait until the suspendProc executed after restart waitProcedure = (WaitProcedure) procExecutor.getProcedure(waitProce); waitProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@Test public void testBatchInsert() throws Exception { final int count = 10; final TestProcedure[] procs = new TestProcedure[count]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestProcedure(i + 1); } procStore.insert(procs); restartAndAssert(count, count, 0, 0); for (int i = 0; i < procs.length; ++i) { final long procId = procs[i].getProcId(); procStore.delete(procId); restartAndAssert(procId != count ? count : 0, count - (i + 1), 0, 0); } procStore.removeInactiveLogsForTesting(); assertEquals("WALs=" + procStore.getActiveLogs(), 1, procStore.getActiveLogs().size()); }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
assertTrue(procCounter.get() >= LAST_PROC_ID); assertTrue(procStore.getStoreTracker().isEmpty()); assertEquals(1, procStore.getActiveLogs().size());
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
assertEquals(1, STORE.getActiveLogs().size()); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); assertEquals(2 + i, STORE.getActiveLogs().size()); EXCHANGER.exchange(Boolean.TRUE); Thread.sleep(1000); UTIL.waitFor(10000, () -> STORE.getActiveLogs().size() <= 2); UTIL.waitFor(10000, () -> EXEC.isFinished(procId));
@Test public void testCompletedProcedure() throws InterruptedException, IOException { long procId = EXEC.submitProcedure(new ExchangeProcedure()); EXCHANGER.exchange(Boolean.FALSE); UTIL.waitFor(10000, () -> EXEC.isFinished(procId)); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); // The exchange procedure is completed but still not deleted yet so we can not delete the // oldest wal file long pid = EXEC.submitProcedure(new NoopNoAckProcedure()); assertEquals(2 + i, STORE.getActiveLogs().size()); UTIL.waitFor(10000, () -> EXEC.isFinished(pid)); } // Only the exchange procedure can not be deleted UTIL.waitFor(10000, () -> EXEC.getCompletedSize() == 1); STORE.rollWriterForTesting(); UTIL.waitFor(10000, () -> STORE.getActiveLogs().size() <= 1); } }
@Test public void testProcIdHoles() throws Exception { // Insert for (int i = 0; i < 100; i += 2) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 10) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); assertEquals(0, loader.getCorruptedCount()); assertEquals((i / 2) + 1, loader.getLoadedCount()); } } assertEquals(10, procStore.getActiveLogs().size()); // Delete for (int i = 0; i < 100; i += 2) { procStore.delete(i); } assertEquals(1, procStore.getActiveLogs().size()); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertEquals(0, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }