@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }
@Test public void testBasicCRUD() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); assertTrue(tracker.isEmpty()); long[] procs = new long[] { 1, 2, 3, 4, 5, 6 }; tracker.insert(procs[0]); tracker.insert(procs[1], new long[] { procs[2], procs[3], procs[4] }); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); tracker.resetModified(); assertFalse(tracker.isAllModified()); for (int i = 0; i < 4; ++i) { tracker.update(procs[i]); assertFalse(tracker.isEmpty()); assertFalse(tracker.isAllModified()); } tracker.update(procs[4]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); tracker.update(procs[5]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); for (int i = 0; i < 5; ++i) { tracker.delete(procs[i]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isAllModified()); } tracker.delete(procs[5]); assertTrue(tracker.isEmpty()); }
@Test public void testPartialTracker() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); tracker.setPartialFlag(true); // nothing in the tracker, the state is unknown assertTrue(tracker.isEmpty()); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579)); // Mark 1 as deleted, now that is a known state tracker.setDeleted(1, true); tracker.dump(); assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(579)); // Mark 579 as non-deleted, now that is a known state tracker.setDeleted(579, false); assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(1)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(2)); assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(579)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(577)); assertEquals(ProcedureStoreTracker.DeleteState.MAYBE, tracker.isDeleted(580)); tracker.setDeleted(579, true); tracker.setPartialFlag(false); assertTrue(tracker.isEmpty()); }
private void setDeletedIfPartial(ProcedureStoreTracker tracker, long procId) { if (tracker.isPartial()) { tracker.setDeleted(procId, true); } }
@Test public void testSeqInsertAndDelete() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); assertTrue(tracker.isEmpty()); tracker.insert(i); assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j)); assertTrue(tracker.isDeleted(j) != ProcedureStoreTracker.DeleteState.NO); tracker.delete(i); assertEquals(ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(j)); assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(j)); assertTrue(tracker.isEmpty());
@Test public void testBasicCRUD() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); assertTrue(tracker.isEmpty()); long[] procs = new long[] { 1, 2, 3, 4, 5, 6 }; tracker.insert(procs[0]); tracker.insert(procs[1], new long[] { procs[2], procs[3], procs[4] }); assertFalse(tracker.isEmpty()); assertTrue(tracker.isUpdated()); tracker.resetUpdates(); assertFalse(tracker.isUpdated()); for (int i = 0; i < 4; ++i) { tracker.update(procs[i]); assertFalse(tracker.isEmpty()); assertFalse(tracker.isUpdated()); } tracker.update(procs[4]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isUpdated()); tracker.update(procs[5]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isUpdated()); for (int i = 0; i < 5; ++i) { tracker.delete(procs[i]); assertFalse(tracker.isEmpty()); assertTrue(tracker.isUpdated()); } tracker.delete(procs[5]); assertTrue(tracker.isEmpty()); }
@Test public void testSetDeletedIfModified() { final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); final long[] procIds = new long[] { 1, 3, 7, 152, 512, 1024, 1025 }; // test single proc for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); for (int i = 0; i < procIds.length; ++i) { tracker.setDeletedIfModified(procIds[i] - 1); tracker.setDeletedIfModified(procIds[i]); tracker.setDeletedIfModified(procIds[i] + 1); } assertEquals(true, tracker.isEmpty()); // test batch tracker.reset(); for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); tracker.setDeletedIfModified(procIds); assertEquals(true, tracker.isEmpty()); }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
@Test public void testDelete() { final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); long[] procIds = new long[] { 65, 1, 193 }; for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); tracker.dump(); } for (int i = 0; i < (64 * 4); ++i) { boolean hasProc = false; for (int j = 0; j < procIds.length; ++j) { if (procIds[j] == i) { hasProc = true; break; } } if (hasProc) { assertEquals(ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(i)); } else { assertEquals("procId=" + i, ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(i)); } } }
private void updateStoreTracker(final PushType type, final long procId, final long[] subProcIds) { switch (type) { case INSERT: if (subProcIds == null) { storeTracker.insert(procId); } else if (procId == Procedure.NO_PROC_ID) { storeTracker.insert(subProcIds); } else { storeTracker.insert(procId, subProcIds); holdingCleanupTracker.setDeletedIfModified(procId); } break; case UPDATE: storeTracker.update(procId); holdingCleanupTracker.setDeletedIfModified(procId); break; case DELETE: if (subProcIds != null && subProcIds.length > 0) { storeTracker.delete(subProcIds); holdingCleanupTracker.setDeletedIfModified(subProcIds); } else { storeTracker.delete(procId); holdingCleanupTracker.setDeletedIfModified(procId); } break; default: throw new RuntimeException("invalid push type " + type); } }
@Test public void testGetActiveMinProcId() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); assertEquals(Procedure.NO_PROC_ID, tracker.getActiveMinProcId()); for (int i = 100; i < 1000; i = 2 * i + 1) { tracker.insert(i); } for (int i = 100; i < 1000; i = 2 * i + 1) { assertEquals(i, tracker.getActiveMinProcId()); tracker.delete(i); } assertEquals(Procedure.NO_PROC_ID, tracker.getActiveMinProcId()); } }
/** * @param active list of active proc ids. To mark them as non-deleted, since by default a proc * id is always marked deleted. */ ProcedureStoreTracker buildTracker(long[] active, long[] updated, long[] deleted) { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (long i : active) { tracker.insert(i); } tracker.resetUpdates(); for (long i : updated) { tracker.update(i); } for (long i : deleted) { tracker.delete(i); } return tracker; }
@Test public void testGetActiveProcIds() { ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int i = 0; i < 10000; i++) { tracker.insert(i * 10); } for (int i = 0; i < 10000; i += 2) { tracker.delete(i * 10); } long[] activeProcIds = tracker.getAllActiveProcIds(); assertEquals(5000, activeProcIds.length); for (int i = 0; i < 5000; i++) { assertEquals((2 * i + 1) * 10, activeProcIds[i]); } }
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
private void updateStoreTracker(final PushType type, final long procId, final long[] subProcIds) { switch (type) { case INSERT: if (subProcIds == null) { storeTracker.insert(procId); } else if (procId == Procedure.NO_PROC_ID) { storeTracker.insert(subProcIds); } else { storeTracker.insert(procId, subProcIds); holdingCleanupTracker.setDeletedIfSet(procId); } break; case UPDATE: storeTracker.update(procId); holdingCleanupTracker.setDeletedIfSet(procId); break; case DELETE: if (subProcIds != null && subProcIds.length > 0) { storeTracker.delete(subProcIds); holdingCleanupTracker.setDeletedIfSet(subProcIds); } else { storeTracker.delete(procId); holdingCleanupTracker.setDeletedIfSet(procId); } break; default: throw new RuntimeException("invalid push type " + type); } }
Loader loader) throws IOException { ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader); tracker.setKeepDeletes(true); if (tracker.isPartial()) { tracker.setPartialFlag(false); tracker.resetModified(); tracker.setKeepDeletes(false);
/** * This method is used when restarting where we need to rebuild the ProcedureStoreTracker. The * {@link #delete(long)} method above assume that the {@link BitSetNode} exists, but when restart * this is not true, as we will read the wal files in reverse order so a delete may come first. */ public void setDeleted(long procId, boolean isDeleted) { BitSetNode node = getOrCreateNode(procId); assert node.contains(procId) : "expected procId=" + procId + " in the node=" + node; node.updateState(procId, isDeleted); trackProcIds(procId); }
private void updateStoreTracker(final PushType type, final long procId, final long[] subProcIds) { switch (type) { case INSERT: if (subProcIds == null) { storeTracker.insert(procId); } else { storeTracker.insert(procId, subProcIds); } break; case UPDATE: storeTracker.update(procId); break; case DELETE: storeTracker.delete(procId); break; default: throw new RuntimeException("invalid push type " + type); } }