private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
log.readTracker(); } catch (IOException e) { log.getTracker().reset(); log.getTracker().setPartialFlag(true); LOG.warn("Unable to read tracker for {}", log, e);
public void read(ProcedureWALFile log) throws IOException { localTracker = log.getTracker(); if (localTracker.isPartial()) { LOG.info("Rebuilding tracker for {}", log);
/** * Tests that tracker for all old logs are loaded back after procedure store is restarted. */ @Test public void trackersLoadedForAllOldLogs() throws Exception { for (int i = 0; i <= 20; ++i) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 5) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); } } assertEquals(5, procStore.getActiveLogs().size()); for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); assertTrue(tracker != null && !tracker.isEmpty()); } }
assertEquals(4, walFiles.size()); LOG.info("Checking wal " + walFiles.get(0)); assertUpdated(walFiles.get(0).getTracker(), procs, new int[]{0, 1, 2, 3}, new int[] {4, 5}); LOG.info("Checking wal " + walFiles.get(1)); assertUpdated(walFiles.get(1).getTracker(), procs, new int[]{2, 3, 4}, new int[] {0, 1, 5}); LOG.info("Checking wal " + walFiles.get(2)); assertUpdated(walFiles.get(2).getTracker(), procs, new int[]{4, 5}, new int[] {0, 1, 2, 3}); LOG.info("Checking global tracker "); assertDeleted(procStore.getStoreTracker(), procs, new int[]{0, 2, 4}, new int[] {1, 3, 5});
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the other WALs are scanned to remove procs already in other wals. // TODO: exit early if holdingCleanupTracker.isEmpty() holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfSet(storeTracker); for (int i = 1, size = logs.size() - 1; i < size; ++i) { holdingCleanupTracker.setDeletedIfSet(logs.get(i).getTracker()); } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. * Otherwise, set storeTracker as partial, and let {@link ProcedureWALFormatReader} rebuild * it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) return; ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
log.readTracker(); } catch (IOException e) { log.getTracker().reset(); log.getTracker().setPartialFlag(true); LOG.warn("Unable to read tracker for " + log + " - " + e.getMessage());
log.readTracker(); } catch (IOException e) { log.getTracker().reset(); log.getTracker().setPartialFlag(true); LOG.warn("Unable to read tracker for {}", log, e);
public void read(final ProcedureWALFile log) throws IOException { localTracker = log.getTracker().isPartial() ? log.getTracker() : null; if (localTracker != null) { LOG.info("Rebuilding tracker for " + log);
public void read(ProcedureWALFile log) throws IOException { localTracker = log.getTracker(); if (localTracker.isPartial()) { LOG.info("Rebuilding tracker for {}", log);
/** * Tests that tracker for all old logs are loaded back after procedure store is restarted. */ @Test public void trackersLoadedForAllOldLogs() throws Exception { for (int i = 0; i <= 20; ++i) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 5) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); } } assertEquals(5, procStore.getActiveLogs().size()); for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); assertTrue(tracker != null && !tracker.isEmpty()); } }
/** * Tests that tracker for all old logs are loaded back after procedure store is restarted. */ @Test public void trackersLoadedForAllOldLogs() throws Exception { for (int i = 0; i <= 20; ++i) { procStore.insert(new TestProcedure(i), null); if (i > 0 && (i % 5) == 0) { LoadCounter loader = new LoadCounter(); storeRestart(loader); } } assertEquals(5, procStore.getActiveLogs().size()); for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); assertTrue(tracker != null && !tracker.isEmpty()); } }
assertEquals(4, walFiles.size()); LOG.info("Checking wal " + walFiles.get(0)); assertUpdated(walFiles.get(0).getTracker(), procs, new int[]{0, 1, 2, 3}, new int[] {4, 5}); LOG.info("Checking wal " + walFiles.get(1)); assertUpdated(walFiles.get(1).getTracker(), procs, new int[]{2, 3, 4}, new int[] {0, 1, 5}); LOG.info("Checking wal " + walFiles.get(2)); assertUpdated(walFiles.get(2).getTracker(), procs, new int[]{4, 5}, new int[] {0, 1, 2, 3}); LOG.info("Checking global tracker "); assertDeleted(procStore.getStoreTracker(), procs, new int[]{0, 2, 4}, new int[] {1, 3, 5});
assertEquals(4, walFiles.size()); LOG.info("Checking wal " + walFiles.get(0)); assertUpdated(walFiles.get(0).getTracker(), procs, new int[]{0, 1, 2, 3}, new int[] {4, 5}); LOG.info("Checking wal " + walFiles.get(1)); assertUpdated(walFiles.get(1).getTracker(), procs, new int[]{2, 3, 4}, new int[] {0, 1, 5}); LOG.info("Checking wal " + walFiles.get(2)); assertUpdated(walFiles.get(2).getTracker(), procs, new int[]{4, 5}, new int[] {0, 1, 2, 3}); LOG.info("Checking global tracker "); assertDeleted(procStore.getStoreTracker(), procs, new int[]{0, 2, 4}, new int[] {1, 3, 5});