final V volume = volumes.get(curVolume); curVolume = (curVolume + 1) % volumes.size(); long availableVolumeSize = volume.getAvailable(); if (availableVolumeSize > blockSize) { curVolumes[curVolumeIndex] = curVolume;
/** * Report a bad block which is hosted on the local DN. * * @param block the bad block which is hosted on the local DN * @param volume the volume that block is stored in and the volume * must not be null * @throws IOException */ public void reportBadBlocks(ExtendedBlock block, FsVolumeSpi volume) throws IOException { BPOfferService bpos = getBPOSForBlock(block); bpos.reportBadBlocks( block, volume.getStorageID(), volume.getStorageType()); }
/** @return is writing to a transient storage? */ public boolean isTransientStorage() { return volume.isTransientStorage(); }
long getRemaining() throws IOException { long remaining = 0L; for (FsVolumeSpi vol : volumes) { try (FsVolumeReference ref = vol.obtainReference()) { remaining += vol.getAvailable(); } catch (ClosedChannelException e) { // ignore } } return remaining; }
/** * Opens all Block pools on a given volume. * * @param source Source * @param poolIters List of PoolIters to maintain. */ private void openPoolIters(FsVolumeSpi source, List<FsVolumeSpi .BlockIterator> poolIters) { Preconditions.checkNotNull(source); Preconditions.checkNotNull(poolIters); for (String blockPoolID : source.getBlockPoolList()) { poolIters.add(source.newBlockIterator(blockPoolID, "DiskBalancerSource")); } }
@Override public String toString() { return "VolumeScanner(" + volume.getBasePath() + ", " + volume.getStorageID() + ")"; }
/** * Allow the scanner to scan the given block pool. * * @param bpid The block pool id. */ public synchronized void enableBlockPoolId(String bpid) { for (BlockIterator iter : blockIters) { if (iter.getBlockPoolId().equals(bpid)) { LOG.warn("{}: already enabled scanning on block pool {}", this, bpid); return; } } BlockIterator iter = null; try { // Load a block iterator for the next block pool on the volume. iter = volume.loadBlockIterator(bpid, BLOCK_ITERATOR_NAME); LOG.trace("{}: loaded block iterator for {}.", this, bpid); } catch (FileNotFoundException e) { LOG.debug("{}: failed to load block iterator: " + e.getMessage(), this); } catch (IOException e) { LOG.warn("{}: failed to load block iterator.", this, e); } if (iter == null) { iter = volume.newBlockIterator(bpid, BLOCK_ITERATOR_NAME); LOG.trace("{}: created new block iterator for {}.", this, bpid); } iter.setMaxStalenessMs(conf.maxStalenessMs); blockIters.add(iter); notify(); }
/** * Get the storageUuid of the volume that stores this replica. */ @Override public String getStorageUuid() { return volume.getStorageID(); }
public void handle(ExtendedBlock block, IOException e) { FsVolumeSpi volume = scanner.volume; if (e == null) { LOG.trace("Successfully scanned {} on {}", block, volume.getBasePath()); return; if (!volume.getDataset().contains(block)) { LOG.debug("Volume {}: block {} is no longer in the dataset.", volume.getBasePath(), block); return; LOG.info("Volume {}: verification failed for {} because of " + "FileNotFoundException. This may be due to a race with write.", volume.getBasePath(), block); return; + volume.getBasePath(), e); try { scanner.datanode.reportBadBlocks(block, volume);
private boolean isReplicaProvided(ReplicaInfo replicaInfo) { if (replicaInfo == null) { return false; } return replicaInfo.getVolume().getStorageType() == StorageType.PROVIDED; }
public void setup(VolumeScanner scanner) { LOG.trace("Starting VolumeScanner {}", scanner.volume.getBasePath()); this.scanner = scanner; }
if (source.isTransientStorage() || dest.isTransientStorage()) { final String errMsg = "Disk Balancer - Unable to support " + "transient storage type."; if (poolIters.size() == 0) { LOG.error("No block pools found on volume. volume : {}. Exiting.", source.getBaseURI()); return; "error count: {}", source.getBaseURI(), dest.getBaseURI(), item.getErrorCount()); break; LOG.info("Copy from {} to {} done. copied {} bytes and {} " + "blocks.", source.getBaseURI(), dest.getBaseURI(), item.getBytesCopied(), item.getBlocksCopied()); this.setExitFlag(); "Dest:{}", source.getBaseURI(), dest.getBaseURI()); this.setExitFlag(); continue; if (dest.getAvailable() > item.getBytesToCopy()) { long begin = System.nanoTime(); this.dataset.moveBlockAcrossVolumes(block, dest); " copyBlocks.", dest.getBaseURI(), block.getNumBytes()); break;
/** * Run this report compiler thread. * * @return the block info report list * @throws IOException if the block pool isn't found */ @Override public ScanInfoPerBlockPool call() throws IOException { String[] bpList = volume.getBlockPoolList(); ScanInfoPerBlockPool result = new ScanInfoPerBlockPool(bpList.length); for (String bpid : bpList) { LinkedList<ScanInfo> report = new LinkedList<>(); File bpFinalizedDir = volume.getFinalizedDir(bpid); perfTimer.start(); throttleTimer.start(); try { result.put(bpid, compileReport(volume, bpFinalizedDir, bpFinalizedDir, report)); } catch (InterruptedException ex) { // Exit quickly and flag the scanner to do the same result = null; break; } } return result; }
HdfsConstants.GRANDFATHER_GENERATION_STAMP; if (vol.getStorageType() == StorageType.PROVIDED) { if (memBlockInfo == null) { if (vol.isTransientStorage()) { ramDiskReplicaTracker.discardReplica(bpid, blockId, true); .build(); volumeMap.add(bpid, diskBlockInfo); if (vol.isTransientStorage()) { long lockedBytesReserved = cacheManager.reserve(diskBlockInfo.getNumBytes()) > 0 ?
DFSTestUtil.getAllBlocks(fs, testFile).get(1).getBlock(); FsVolumeSpi volumeWithBlock = dn.getFSDataset().getVolume(block); String basePath = volumeWithBlock.getBasePath(); File storageDir = new File(basePath); URI fileUri = storageDir.toURI(); String dirWithBlock = "[" + volumeWithBlock.getStorageType() + "]" + fileUri; String newDirs = dirWithBlock; for (String dir : oldDirs) {
/** * Returns volume UUID to volume base path map. * * @return Map * @throws DiskBalancerException */ private Map<String, String> getStorageIDToVolumeBasePathMap() throws DiskBalancerException { Map<String, String> storageIDToVolBasePathMap = new HashMap<>(); FsDatasetSpi.FsVolumeReferences references; try { try(AutoCloseableLock lock = this.dataset.acquireDatasetLock()) { references = this.dataset.getFsVolumeReferences(); for (int ndx = 0; ndx < references.size(); ndx++) { FsVolumeSpi vol = references.get(ndx); storageIDToVolBasePathMap.put(vol.getStorageID(), vol.getBaseURI().getPath()); } references.close(); } } catch (IOException ex) { LOG.error("Disk Balancer - Internal Error.", ex); throw new DiskBalancerException("Internal error", ex, DiskBalancerException.Result.INTERNAL_ERROR); } return storageIDToVolBasePathMap; }
/** Create a metafile in a random volume*/ private long createMetaFile() throws IOException { List<? extends FsVolumeSpi> volumes = fds.getVolumes(); int index = rand.nextInt(volumes.size() - 1); long id = getFreeBlockId(); File finalizedDir = volumes.get(index).getFinalizedDir(bpid); File file = new File(finalizedDir, getMetaFile(id)); if (file.createNewFile()) { LOG.info("Created metafile " + file.getName()); } return id; }
/** * Run this report compiler thread. * * @return the block info report list * @throws IOException if the block pool isn't found */ @Override public ScanInfoPerBlockPool call() throws IOException { String[] bpList = volume.getBlockPoolList(); ScanInfoPerBlockPool result = new ScanInfoPerBlockPool(bpList.length); perfTimer.start(); throttleTimer.start(); for (String bpid : bpList) { LinkedList<ScanInfo> report = new LinkedList<>(); perfTimer.reset().start(); throttleTimer.reset().start(); try { result.put(bpid, volume.compileReport(bpid, report, this)); } catch (InterruptedException ex) { // Exit quickly and flag the scanner to do the same result = null; break; } } return result; }
if (!volume.getDataset().contains(block)) { LOG.debug("Volume {}: block {} is no longer in the dataset.", volume, block);
@Override public String toString() { return "VolumeScanner(" + volume.getBasePath() + ", " + volume.getStorageID() + ")"; }