@Test public void testWalCleanerSequentialClean() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(logs.size(), i + 2); // Extra 1 for current ongoing wal. } // Delete procedures in sequential order make sure that only the corresponding wal is deleted // from logs list. final int[] deleteOrder = new int[] { 0, 1, 2, 3, 4 }; for (int i = 0; i < deleteOrder.length; i++) { procStore.delete(procs[deleteOrder[i]].getProcId()); procStore.removeInactiveLogsForTesting(); assertFalse(logs.get(deleteOrder[i]).toString(), procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); assertEquals(procStore.getActiveLogs().size(), procs.length - i); } }
Procedure<?> proc1 = new TestSequentialProcedure(); procIds.add(proc1.getProcId()); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); Procedure<?>[] child2 = new Procedure[2]; child2[0] = new TestSequentialProcedure(); child2[1] = new TestSequentialProcedure();
final Procedure<?>[] procs = new Procedure[6]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestSequentialProcedure();
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testCorruptedEntries() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove some byte from the log // (enough to cut the trailer and corrupt some entries) FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 1823); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertTrue(procStore.getCorruptedLogs() != null); assertEquals(1, procStore.getCorruptedLogs().size()); assertEquals(87, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testCorruptedEntries() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove some byte from the log // (enough to cut the trailer and corrupt some entries) FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 1823); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertTrue(procStore.getCorruptedLogs() != null); assertEquals(1, procStore.getCorruptedLogs().size()); assertEquals(87, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testCorruptedTrailer() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove 4 byte from the trailer FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 4); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertEquals(100, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testCorruptedTrailer() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove 4 byte from the trailer FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 4); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertEquals(100, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testNoTrailerDoubleRestart() throws Exception { Procedure<?> proc0 = new TestSequentialProcedure(); procStore.insert(proc0, null); Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); procStore.rollWriterForTesting();
@Test public void testWalCleanerWithEmptyRolls() throws Exception { final Procedure<?>[] procs = new Procedure[3]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); } assertEquals(1, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(3, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting(); procStore.rollWriterForTesting(); if (i < (procs.length - 1)) { assertEquals(3 + ((i + 1) * 2), procStore.getActiveLogs().size()); } } assertEquals(7, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.delete(procs[i].getProcId()); assertEquals(7 - ((i + 1) * 2), procStore.getActiveLogs().size()); } assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testCorruptedEntries() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove some byte from the log // (enough to cut the trailer and corrupt some entries) FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 1823); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertTrue(procStore.getCorruptedLogs() != null); assertEquals(1, procStore.getCorruptedLogs().size()); assertEquals(87, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testWalCleanerNoHoles() throws Exception { final Procedure<?>[] procs = new Procedure[5]; ArrayList<ProcedureWALFile> logs = null; // Insert procedures and roll wal after every insert. for (int i = 0; i < procs.length; i++) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); logs = procStore.getActiveLogs(); assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. } for (int i = 1; i < procs.length; i++) { procStore.delete(procs[i].getProcId()); } assertEquals(procs.length + 1, procStore.getActiveLogs().size()); procStore.delete(procs[0].getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testCorruptedTrailer() throws Exception { // Insert something for (int i = 0; i < 100; ++i) { procStore.insert(new TestSequentialProcedure(), null); } // Stop the store procStore.stop(false); // Remove 4 byte from the trailer FileStatus[] logs = fs.listStatus(logDir); assertEquals(1, logs.length); corruptLog(logs[0], 4); LoadCounter loader = new LoadCounter(); storeRestart(loader); assertEquals(100, loader.getLoadedCount()); assertEquals(0, loader.getCorruptedCount()); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }