try { long monotonicMs = Time.monotonicNow(); expireOldScannedBytesRecords(monotonicMs); if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec, scannedBytesSum, startMinute, curMinute)) { } else { if ((curBlockIter == null) || curBlockIter.atEnd()) { long timeout = findNextUsableBlockIter(); if (timeout > 0) { LOG.trace("{}: no block pools are ready to scan yet. Waiting " + saveBlockIterator(curBlockIter); return 0; LOG.debug("{}: saving block iterator {} after {} ms.", this, curBlockIter, saveDelta); saveBlockIterator(curBlockIter); bytesScanned = scanBlock(block, conf.targetBytesPerSec); if (bytesScanned >= 0) { scannedBytesSum += bytesScanned;
scanner = new VolumeScanner(conf, datanode, ref); scanner.start(); scanners.put(volume.getStorageID(), scanner); success = true;
/** * Disable scanning a given block pool id. * * @param bpid The block pool id to disable scanning for. */ synchronized void disableBlockPoolId(String bpid) { Preconditions.checkNotNull(bpid); for (VolumeScanner scanner : scanners.values()) { scanner.disableBlockPoolId(bpid); } }
suspectBlock = popNextSuspectBlock(); timeout = runLoop(suspectBlock); saveBlockIterator(iter); IOUtils.cleanup(null, iter);
@Test(timeout=120000) public void testCalculateNeededBytesPerSec() throws Exception { // If we didn't check anything the last hour, we should scan now. Assert.assertTrue( VolumeScanner.calculateShouldScan("test", 100, 0, 0, 60)); // If, on average, we checked 101 bytes/s checked during the last hour, // stop checking now. Assert.assertFalse(VolumeScanner. calculateShouldScan("test", 100, 101 * 3600, 1000, 5000)); // Target is 1 byte / s, but we didn't scan anything in the last minute. // Should scan now. Assert.assertTrue(VolumeScanner. calculateShouldScan("test", 1, 3540, 0, 60)); // Target is 1000000 byte / s, but we didn't scan anything in the last // minute. Should scan now. Assert.assertTrue(VolumeScanner. calculateShouldScan("test", 100000L, 354000000L, 0, 60)); Assert.assertFalse(VolumeScanner. calculateShouldScan("test", 100000L, 365000000L, 0, 60)); }
/** * Enable scanning a given block pool id. * * @param bpid The block pool id to enable scanning for. */ synchronized void enableBlockPoolId(String bpid) { Preconditions.checkNotNull(bpid); for (VolumeScanner scanner : scanners.values()) { scanner.enableBlockPoolId(bpid); } }
@VisibleForTesting synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) { VolumeScanner scanner = scanners.get(volumeId); if (scanner == null) { return null; } return scanner.getStatistics(); }
/** * Shut down this scanner. */ public synchronized void shutdown() { stopping = true; notify(); this.interrupt(); }
/** * Mark a block as "suspect." * * This means that we should try to rescan it soon. Note that the * VolumeScanner keeps a list of recently suspicious blocks, which * it uses to avoid rescanning the same block over and over in a short * time frame. * * @param storageId The ID of the storage where the block replica * is being stored. * @param block The block's ID and block pool id. */ synchronized void markSuspectBlock(String storageId, ExtendedBlock block) { if (!isEnabled()) { LOG.debug("Not scanning suspicious block {} on {}, because the block " + "scanner is disabled.", block, storageId); return; } VolumeScanner scanner = scanners.get(storageId); if (scanner == null) { // This could happen if the volume is in the process of being removed. // The removal process shuts down the VolumeScanner, but the volume // object stays around as long as there are references to it (which // should not be that long.) LOG.info("Not scanning suspicious block {} on {}, because there is no " + "volume scanner for that storageId.", block, storageId); return; } scanner.markSuspectBlock(block); }
suspectBlock = popNextSuspectBlock(); timeout = runLoop(suspectBlock); saveBlockIterator(iter); IOUtils.cleanup(null, iter);
/** * Enable scanning a given block pool id. * * @param bpid The block pool id to enable scanning for. */ synchronized void enableBlockPoolId(String bpid) { Preconditions.checkNotNull(bpid); for (VolumeScanner scanner : scanners.values()) { scanner.enableBlockPoolId(bpid); } }
@VisibleForTesting synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) { VolumeScanner scanner = scanners.get(volumeId); if (scanner == null) { return null; } return scanner.getStatistics(); }
/** * Shut down this scanner. */ public synchronized void shutdown() { stopping = true; notify(); this.interrupt(); }
/** * Mark a block as "suspect." * * This means that we should try to rescan it soon. Note that the * VolumeScanner keeps a list of recently suspicious blocks, which * it uses to avoid rescanning the same block over and over in a short * time frame. * * @param storageId The ID of the storage where the block replica * is being stored. * @param block The block's ID and block pool id. */ synchronized void markSuspectBlock(String storageId, ExtendedBlock block) { if (!isEnabled()) { LOG.debug("Not scanning suspicious block {} on {}, because the block " + "scanner is disabled.", block, storageId); return; } VolumeScanner scanner = scanners.get(storageId); if (scanner == null) { // This could happen if the volume is in the process of being removed. // The removal process shuts down the VolumeScanner, but the volume // object stays around as long as there are references to it (which // should not be that long.) LOG.info("Not scanning suspicious block {} on {}, because there is no " + "volume scanner for that storageId.", block, storageId); return; } scanner.markSuspectBlock(block); }
try { long monotonicMs = Time.monotonicNow(); expireOldScannedBytesRecords(monotonicMs); if (!calculateShouldScan(volume.getStorageID(), conf.targetBytesPerSec, scannedBytesSum, startMinute, curMinute)) { } else { if ((curBlockIter == null) || curBlockIter.atEnd()) { long timeout = findNextUsableBlockIter(); if (timeout > 0) { LOG.trace("{}: no block pools are ready to scan yet. Waiting " + saveBlockIterator(curBlockIter); return 0; LOG.debug("{}: saving block iterator {} after {} ms.", this, curBlockIter, saveDelta); saveBlockIterator(curBlockIter); bytesScanned = scanBlock(block, conf.targetBytesPerSec); if (bytesScanned >= 0) { scannedBytesSum += bytesScanned;
suspectBlock = popNextSuspectBlock(); timeout = runLoop(suspectBlock); saveBlockIterator(iter); IOUtils.cleanup(null, iter);
scanner = new VolumeScanner(conf, datanode, ref); scanner.start(); scanners.put(volume.getStorageID(), scanner); success = true;
/** * Enable scanning a given block pool id. * * @param bpid The block pool id to enable scanning for. */ synchronized void enableBlockPoolId(String bpid) { Preconditions.checkNotNull(bpid); for (VolumeScanner scanner : scanners.values()) { scanner.enableBlockPoolId(bpid); } }
/** * Disable scanning a given block pool id. * * @param bpid The block pool id to disable scanning for. */ synchronized void disableBlockPoolId(String bpid) { Preconditions.checkNotNull(bpid); for (VolumeScanner scanner : scanners.values()) { scanner.disableBlockPoolId(bpid); } }
@VisibleForTesting synchronized VolumeScanner.Statistics getVolumeStats(String volumeId) { VolumeScanner scanner = scanners.get(volumeId); if (scanner == null) { return null; } return scanner.getStatistics(); }