/** * 3 possibilities * - If maxLastModifiedInterval <= 0 then return 0 which is interpreted as current by delete call * (For testing purposes only) * - If oldest checkpoint creation date > 0 then reference time is the earliest of that and the parameter * maxModificationReferenceTime * - Else the parameter maxModificationReferenceTime is used as the reference time * * @param maxModificationReferenceTime typically the mark phase start time (could be 0 for tests) * @return max modified time of blobs to be considered for deletion */ private long getMaxModifiedTime(long maxModificationReferenceTime) { if (maxLastModifiedInterval <= 0) { return 0; } long oldestCheckpoint = -1; if (checkpointMbean != null) { oldestCheckpoint = checkpointMbean.getOldestCheckpointCreationDate().getTime(); LOG.debug("Oldest checkpoint data retrieved {} ", oldestCheckpoint); } LOG.debug("maxModificationReferenceTime {} ", maxModificationReferenceTime); maxModificationReferenceTime = maxModificationReferenceTime <= 0 ? System.currentTimeMillis() : maxModificationReferenceTime; long calculatedReferenceTime = (oldestCheckpoint <= 0 ? maxModificationReferenceTime : Math.min(maxModificationReferenceTime, oldestCheckpoint)); LOG.debug("Calculated reference time {} ", calculatedReferenceTime); return (calculatedReferenceTime - maxLastModifiedInterval); }
private long getOldestCheckpointCreationTimestamp() { Tracker<CheckpointMBean> tracker = whiteboard.track(CheckpointMBean.class); try { List<CheckpointMBean> services = tracker.getServices(); if (services.size() == 1) { return services.get(0).getOldestCheckpointCreationTimestamp(); } else if (services.isEmpty()) { LOG.warn("Unable to get checkpoint mbean. No service of required type found."); return -1; } else { LOG.warn("Unable to get checkpoint mbean. Multiple services of required type found."); return -1; } } finally { tracker.stop(); } } }
@Test public void gcCheckpointHeldMaxAgeChange() throws Exception { log.info("Staring gcCheckpointHeldMaxAgeChange()"); long afterSetupTime = clock.getTime(); log.info("{} afterSetupTime time", afterSetupTime); checkpointMBean.createCheckpoint(100); Set<String> afterCheckpointBlobs = createBlobs(cluster.blobStore, 2, 100); cluster.blobStoreState.blobsPresent.addAll(afterCheckpointBlobs); log.info("{} blobs added : {}", cluster.blobStoreState.blobsAdded.size(), cluster.blobStoreState.blobsAdded); log.info("{} blobs remaining : {}", cluster.blobStoreState.blobsPresent.size(), cluster.blobStoreState.blobsPresent); long maxGcAge = checkpointMBean.getOldestCheckpointCreationTimestamp() - afterSetupTime; log.info("Max age configured {}", maxGcAge); Set<String> existingAfterGC = executeGarbageCollection(cluster, cluster.getCollector(maxGcAge), false); assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsPresent, existingAfterGC).isEmpty()); }
@Test public void gcCheckpointHeld() throws Exception { log.info("Staring gcCheckpointHeld()"); long afterSetupTime = clock.getTime(); log.info("afterSetupTime {}", afterSetupTime); checkpointMBean.createCheckpoint(100); Set<String> afterCheckpointBlobs = createBlobs(cluster.blobStore, 2, 100); Set<String> present = Sets.union(cluster.blobStoreState.blobsPresent, afterCheckpointBlobs); long maxGcAge = checkpointMBean.getOldestCheckpointCreationTimestamp() - afterSetupTime; log.info("{} blobs remaining : {}", present.size(), present); Set<String> existingAfterGC = executeGarbageCollection(cluster, cluster.getCollector(maxGcAge), false); assertTrue(Sets.symmetricDifference(present, existingAfterGC).isEmpty()); }
@Test public void gcCheckpointHeldNoAddition() throws Exception { log.info("Staring gcCheckpointHeldNoAddition()"); long afterSetupTime = clock.getTime(); log.info("afterSetupTime {}", afterSetupTime); checkpointMBean.createCheckpoint(100); long maxGcAge = checkpointMBean.getOldestCheckpointCreationTimestamp() - afterSetupTime; Set<String> existingAfterGC = executeGarbageCollection(cluster, cluster.getCollector(maxGcAge), false); assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsPresent, existingAfterGC).isEmpty()); }
/** * 3 possibilities * - If maxLastModifiedInterval <= 0 then return 0 which is interpreted as current by delete call * (For testing purposes only) * - If oldest checkpoint creation date > 0 then reference time is the earliest of that and the parameter * maxModificationReferenceTime * - Else the parameter maxModificationReferenceTime is used as the reference time * * @param maxModificationReferenceTime typically the mark phase start time (could be 0 for tests) * @return max modified time of blobs to be considered for deletion */ private long getMaxModifiedTime(long maxModificationReferenceTime) { if (maxLastModifiedInterval <= 0) { return 0; } long oldestCheckpoint = -1; if (checkpointMbean != null) { oldestCheckpoint = checkpointMbean.getOldestCheckpointCreationDate().getTime(); LOG.debug("Oldest checkpoint data retrieved {} ", oldestCheckpoint); } LOG.debug("maxModificationReferenceTime {} ", maxModificationReferenceTime); maxModificationReferenceTime = maxModificationReferenceTime <= 0 ? System.currentTimeMillis() : maxModificationReferenceTime; long calculatedReferenceTime = (oldestCheckpoint <= 0 ? maxModificationReferenceTime : Math.min(maxModificationReferenceTime, oldestCheckpoint)); LOG.debug("Calculated reference time {} ", calculatedReferenceTime); return (calculatedReferenceTime - maxLastModifiedInterval); }
private long getOldestCheckpointCreationTimestamp() { Tracker<CheckpointMBean> tracker = whiteboard.track(CheckpointMBean.class); try { List<CheckpointMBean> services = tracker.getServices(); if (services.size() == 1) { return services.get(0).getOldestCheckpointCreationTimestamp(); } else if (services.isEmpty()) { LOG.warn("Unable to get checkpoint mbean. No service of required type found."); return -1; } else { LOG.warn("Unable to get checkpoint mbean. Multiple services of required type found."); return -1; } } finally { tracker.stop(); } } }
/** * 3 possibilities * - If maxLastModifiedInterval <= 0 then return 0 which is interpreted as current by delete call * (For testing purposes only) * - If oldest checkpoint creation date > 0 then reference time is the earliest of that and the parameter * maxModificationReferenceTime * - Else the parameter maxModificationReferenceTime is used as the reference time * * @param maxModificationReferenceTime typically the mark phase start time (could be 0 for tests) * @return max modified time of blobs to be considered for deletion */ private long getMaxModifiedTime(long maxModificationReferenceTime) { if (maxLastModifiedInterval <= 0) { return 0; } long oldestCheckpoint = -1; if (checkpointMbean != null) { oldestCheckpoint = checkpointMbean.getOldestCheckpointCreationDate().getTime(); LOG.debug("Oldest checkpoint data retrieved {} ", oldestCheckpoint); } LOG.debug("maxModificationReferenceTime {} ", maxModificationReferenceTime); maxModificationReferenceTime = maxModificationReferenceTime <= 0 ? System.currentTimeMillis() : maxModificationReferenceTime; long calculatedReferenceTime = (oldestCheckpoint <= 0 ? maxModificationReferenceTime : Math.min(maxModificationReferenceTime, oldestCheckpoint)); LOG.debug("Calculated reference time {} ", calculatedReferenceTime); return (calculatedReferenceTime - maxLastModifiedInterval); }