@Override public void run() { Random rand = new Random(); TestProcedure proc; do { // After HBASE-15579 there may be gap in the procId sequence, trying to simulate that. long procId = procCounter.addAndGet(1 + rand.nextInt(3)); proc = new TestProcedure(procId); // Insert procStore.insert(proc, null); // Update for (int i = 0, nupdates = rand.nextInt(10); i <= nupdates; ++i) { try { Thread.sleep(0, rand.nextInt(15)); } catch (InterruptedException e) {} procStore.update(proc); } // Delete procStore.delete(proc.getProcId()); } while (proc.getProcId() < LAST_PROC_ID); } };
proc.setData(serializedState); store.insert(proc, null); store.update(proc);
private void writeWals() throws IOException { List<Integer> procStates = shuffleProcWriteSequence(); TestProcedure[] procs = new TestProcedure[numProcs + 1]; // 0 is not used. int numProcsPerWal = numWals > 0 ? procStates.size() / numWals : Integer.MAX_VALUE; long startTime = currentTimeMillis(); long lastTime = startTime; for (int i = 0; i < procStates.size(); ++i) { int procId = procStates.get(i); if (procId < 0) { store.delete(procs[-procId].getProcId()); procs[-procId] = null; } else if (procs[procId] == null) { procs[procId] = new TestProcedure(procId, 0); procs[procId].setData(serializedState); store.insert(procs[procId], null); } else { store.update(procs[procId]); } if (i > 0 && i % numProcsPerWal == 0) { long currentTime = currentTimeMillis(); System.out.println("Forcing wall roll. Time taken on last WAL: " + (currentTime - lastTime) / 1000.0f + " sec"); store.rollWriterForTesting(); lastTime = currentTime; } } long timeTaken = currentTimeMillis() - startTime; System.out.println("\n\nDone writing WALs.\nNum procs : " + numProcs + "\nTotal time taken : " + StringUtils.humanTimeDiff(timeTaken) + "\n\n"); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testLoadChildren() throws Exception { TestProcedure a = new TestProcedure(1, 0); TestProcedure b = new TestProcedure(2, 1); TestProcedure c = new TestProcedure(3, 1); // INIT procStore.insert(a, null); // Run A first step a.addStackId(0); procStore.update(a); // Run A second step a.addStackId(1); procStore.insert(a, new Procedure[] { b, c }); // Run B first step b.addStackId(2); procStore.update(b); // Run C first and last step c.addStackId(3); procStore.update(c); // Run B second setp b.addStackId(4); procStore.update(b); // back to A a.addStackId(5); a.setSuccessState(); procStore.delete(a, new long[] { b.getProcId(), c.getProcId() }); restartAndAssert(3, 0, 1, 0); }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
@Test public void testWalCleanerWithEmptyRolls() throws Exception { final Procedure<?>[] procs = new Procedure[3]; for (int i = 0; i < procs.length; ++i) { procs[i] = new TestSequentialProcedure(); procStore.insert(procs[i], null); } assertEquals(1, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); procStore.rollWriterForTesting(); assertEquals(3, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting(); procStore.rollWriterForTesting(); if (i < (procs.length - 1)) { assertEquals(3 + ((i + 1) * 2), procStore.getActiveLogs().size()); } } assertEquals(7, procStore.getActiveLogs().size()); for (int i = 0; i < procs.length; ++i) { procStore.delete(procs[i].getProcId()); assertEquals(7 - ((i + 1) * 2), procStore.getActiveLogs().size()); } assertEquals(1, procStore.getActiveLogs().size()); }
procStore.rollWriterForTesting(); procStore.delete(procs[2].getProcId()); procStore.update(procs[3]); procStore.insert(procs[4], null); procStore.rollWriterForTesting();
procStore.update(proc2); procStore.rollWriterForTesting();
procStore.update(procs[i]); procStore.rollWriterForTesting();
@Override public void run() { Random rand = new Random(); TestProcedure proc; do { // After HBASE-15579 there may be gap in the procId sequence, trying to simulate that. long procId = procCounter.addAndGet(1 + rand.nextInt(3)); proc = new TestProcedure(procId); // Insert procStore.insert(proc, null); // Update for (int i = 0, nupdates = rand.nextInt(10); i <= nupdates; ++i) { try { Thread.sleep(0, rand.nextInt(15)); } catch (InterruptedException e) {} procStore.update(proc); } // Delete procStore.delete(proc.getProcId()); } while (proc.getProcId() < LAST_PROC_ID); } };
@Override public void run() { Random rand = new Random(); TestProcedure proc; do { // After HBASE-15579 there may be gap in the procId sequence, trying to simulate that. long procId = procCounter.addAndGet(1 + rand.nextInt(3)); proc = new TestProcedure(procId); // Insert procStore.insert(proc, null); // Update for (int i = 0, nupdates = rand.nextInt(10); i <= nupdates; ++i) { try { Thread.sleep(0, rand.nextInt(15)); } catch (InterruptedException e) {} procStore.update(proc); } // Delete procStore.delete(proc.getProcId()); } while (proc.getProcId() < LAST_PROC_ID); } };
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); // generates first log with p1 + p2 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); procStore.update(p2); procStore.rollWriterForTesting(); // generates second log with p2 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); procStore.update(p2); procStore.rollWriterForTesting(); // generates third log with p2 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. assertEquals(4, procStore.getActiveLogs().size()); procStore.update(p1); procStore.rollWriterForTesting(); // generates fourth log with p1 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. assertEquals(3, procStore.getActiveLogs().size()); assertFalse(procStore.getActiveLogs().contains(log1)); assertFalse(procStore.getActiveLogs().contains(log2)); }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }