/** * Resets internal state to same as given {@code tracker}, and change the deleted flag according * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap. * <p/> * The {@code resetDelete} will be set to true when building cleanup tracker, please see the * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the * deleted flag if {@code resetDelete} is true. */ public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) { reset(); // resetDelete will true if we are building the cleanup tracker, as we will reset deleted flags // for all the unmodified bits to 1, the partial flag is useless so set it to false for not // confusing the developers when debugging. this.partial = resetDelete ? false : tracker.partial; this.minModifiedProcId = tracker.minModifiedProcId; this.maxModifiedProcId = tracker.maxModifiedProcId; this.keepDeletes = tracker.keepDeletes; for (Map.Entry<Long, BitSetNode> entry : tracker.map.entrySet()) { map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete)); } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) { reset(); for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) { final BitSetNode node = new BitSetNode(protoNode); map.put(node.getStart(), node); } }
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
log.readTracker(); } catch (IOException e) { log.getTracker().reset(); log.getTracker().setPartialFlag(true); LOG.warn("Unable to read tracker for {}", log, e);
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the other WALs are scanned to remove procs already in other wals. // TODO: exit early if holdingCleanupTracker.isEmpty() holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfSet(storeTracker); for (int i = 1, size = logs.size() - 1; i < size; ++i) { holdingCleanupTracker.setDeletedIfSet(logs.get(i).getTracker()); } }
@Test public void testRandLoad() { final int NPROCEDURES = 2500; final int NRUNS = 5000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); Random rand = new Random(1); for (int i = 0; i < NRUNS; ++i) { assertTrue(tracker.isEmpty()); int count = 0; while (count < NPROCEDURES) { long procId = rand.nextLong(); if (procId < 1) continue; tracker.setDeleted(procId, i % 2 == 0); count++; } tracker.reset(); } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }
/** * If last log's tracker is not null, use it as {@link #storeTracker}. * Otherwise, set storeTracker as partial, and let {@link ProcedureWALFormatReader} rebuild * it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) return; ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
@Test public void testSetDeletedIfModified() { final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); final long[] procIds = new long[] { 1, 3, 7, 152, 512, 1024, 1025 }; // test single proc for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); for (int i = 0; i < procIds.length; ++i) { tracker.setDeletedIfModified(procIds[i] - 1); tracker.setDeletedIfModified(procIds[i]); tracker.setDeletedIfModified(procIds[i] + 1); } assertEquals(true, tracker.isEmpty()); // test batch tracker.reset(); for (int i = 0; i < procIds.length; ++i) { tracker.insert(procIds[i]); } assertEquals(false, tracker.isEmpty()); tracker.setDeletedIfModified(procIds); assertEquals(true, tracker.isEmpty()); }
/** * Resets internal state to same as given {@code tracker}, and change the deleted flag according * to the modified flag if {@code resetDelete} is true. Does deep copy of the bitmap. * <p/> * The {@code resetDelete} will be set to true when building cleanup tracker, please see the * comments in {@link BitSetNode#BitSetNode(BitSetNode, boolean)} to learn how we change the * deleted flag if {@code resetDelete} is true. */ public void resetTo(ProcedureStoreTracker tracker, boolean resetDelete) { reset(); // resetDelete will true if we are building the cleanup tracker, as we will reset deleted flags // for all the unmodified bits to 1, the partial flag is useless so set it to false for not // confusing the developers when debugging. this.partial = resetDelete ? false : tracker.partial; this.minModifiedProcId = tracker.minModifiedProcId; this.maxModifiedProcId = tracker.maxModifiedProcId; this.keepDeletes = tracker.keepDeletes; for (Map.Entry<Long, BitSetNode> entry : tracker.map.entrySet()) { map.put(entry.getKey(), new BitSetNode(entry.getValue(), resetDelete)); } }
private void initTrackerFromOldLogs() { // TODO: Load the most recent tracker available if (!logs.isEmpty()) { ProcedureWALFile log = logs.getLast(); try { log.readTracker(storeTracker); } catch (IOException e) { LOG.warn("Unable to read tracker for " + log + " - " + e.getMessage()); // try the next one... storeTracker.reset(); storeTracker.setPartialFlag(true); } } }
public void resetToProto(final ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) { reset(); for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) { final BitSetNode node = new BitSetNode(protoNode); map.put(node.getStart(), node); } }
public void resetToProto(ProcedureProtos.ProcedureStoreTracker trackerProtoBuf) { reset(); for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: trackerProtoBuf.getNodeList()) { final BitSetNode node = new BitSetNode(protoNode); map.put(node.getStart(), node); } }
public void readFrom(final InputStream stream) throws IOException { reset(); final ProcedureProtos.ProcedureStoreTracker data = ProcedureProtos.ProcedureStoreTracker.parseDelimitedFrom(stream); for (ProcedureProtos.ProcedureStoreTracker.TrackerNode protoNode: data.getNodeList()) { final BitSetNode node = BitSetNode.convert(protoNode); map.put(node.getStart(), node); } } }
@Test public void testRandLoad() { final int NPROCEDURES = 2500; final int NRUNS = 5000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); Random rand = new Random(1); for (int i = 0; i < NRUNS; ++i) { assertTrue(tracker.isEmpty()); int count = 0; while (count < NPROCEDURES) { long procId = rand.nextLong(); if (procId < 1) continue; tracker.setDeleted(procId, i % 2 == 0); count++; } tracker.reset(); } }
@Test public void testRandLoad() { final int NPROCEDURES = 2500; final int NRUNS = 5000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); Random rand = new Random(1); for (int i = 0; i < NRUNS; ++i) { assertTrue(tracker.isEmpty()); int count = 0; while (count < NPROCEDURES) { long procId = rand.nextLong(); if (procId < 1) continue; tracker.setDeleted(procId, i % 2 == 0); count++; } tracker.reset(); } }
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }
@Test public void testLoad() { final int MAX_PROCS = 1000; final ProcedureStoreTracker tracker = new ProcedureStoreTracker(); for (int numProcs = 1; numProcs < MAX_PROCS; ++numProcs) { for (int start = 1; start <= numProcs; ++start) { assertTrue(tracker.isEmpty()); LOG.debug("loading " + numProcs + " procs from start=" + start); for (int i = start; i <= numProcs; ++i) { tracker.setDeleted(i, false); } for (int i = 1; i < start; ++i) { tracker.setDeleted(i, false); } tracker.reset(); } } }