/** * Returns the next block that we copy from all the block pools. This * function looks across all block pools to find the next block to copy. * * @param poolIters - List of BlockIterators * @return ExtendedBlock. */ ExtendedBlock getNextBlock(List<FsVolumeSpi.BlockIterator> poolIters, DiskBalancerWorkItem item) { Preconditions.checkNotNull(poolIters); int currentCount = 0; ExtendedBlock block = null; while (block == null && currentCount < poolIters.size()) { currentCount++; int index = poolIndex++ % poolIters.size(); FsVolumeSpi.BlockIterator currentPoolIter = poolIters.get(index); block = getBlockToCopy(currentPoolIter, item); } if (block == null) { try { item.setErrMsg("No source blocks found to move."); LOG.error("No movable source blocks found. {}", item.toJson()); } catch (IOException e) { LOG.error("Unable to get json from Item."); } } return block; }
/** * Computes sleep delay needed based on the block that just got copied. we * copy using a burst mode, that is we let the copy proceed in full * throttle. Once a copy is done, we compute how many bytes have been * transferred and try to average it over the user specified bandwidth. In * other words, This code implements a poor man's token bucket algorithm for * traffic shaping. * * @param bytesCopied - byteCopied. * @param timeUsed in milliseconds * @param item DiskBalancerWorkItem * @return sleep delay in Milliseconds. */ private long computeDelay(long bytesCopied, long timeUsed, DiskBalancerWorkItem item) { // we had an overflow, ignore this reading and continue. if (timeUsed == 0) { return 0; } final int megaByte = 1024 * 1024; long bytesInMB = bytesCopied / megaByte; long lastThroughput = bytesInMB / SECONDS.convert(timeUsed, TimeUnit.MILLISECONDS); long delay = (bytesInMB / getDiskBandwidth(item)) - lastThroughput; return (delay <= 0) ? 0 : MILLISECONDS.convert(delay, TimeUnit.SECONDS); }
openPoolIters(source, poolIters); if (poolIters.size() == 0) { LOG.error("No block pools found on volume. volume : {}. Exiting.", while (shouldRun()) { try { if (item.getErrorCount() > getMaxError(item)) { LOG.error("Exceeded the max error count. source {}, dest: {} " + "error count: {}", source.getBaseURI(), if (isCloseEnough(item)) { LOG.info("Copy from {} to {} done. copied {} bytes and {} " + "blocks.", source.getBaseURI(), dest.getBaseURI(), item.getBytesCopied(), item.getBlocksCopied()); this.setExitFlag(); continue; ExtendedBlock block = getNextBlock(poolIters, item); this.setExitFlag(); continue; if (!shouldRun()) { continue; Thread.sleep(computeDelay(block.getNumBytes(), timeUsed, item));
while (!iter.atEnd() && item.getErrorCount() < getMaxError(item)) { try { ExtendedBlock block = iter.nextBlock(); if (isLessThanNeeded(block.getNumBytes(), item)) { return block; if (item.getErrorCount() >= getMaxError(item)) { item.setErrMsg("Error count exceeded."); LOG.info("Maximum error count exceeded. Error count: {} Max error:{} ",
/** * Initilizes {@link DiskBalancer}. * @param data - FSDataSet * @param conf - Config */ private void initDiskBalancer(FsDatasetSpi data, Configuration conf) { if (this.diskBalancer != null) { return; } DiskBalancer.BlockMover mover = new DiskBalancer.DiskBalancerMover(data, conf); this.diskBalancer = new DiskBalancer(getDatanodeUuid(), conf, mover); }
/** * Inflates bytesCopied and returns true or false. This allows us to stop * copying if we have reached close enough. * * @param item DiskBalancerWorkItem * @return -- false if we need to copy more, true if we are done */ private boolean isCloseEnough(DiskBalancerWorkItem item) { long temp = item.getBytesCopied() + ((item.getBytesCopied() * getBlockTolerancePercentage(item)) / 100); return (item.getBytesToCopy() >= temp) ? false : true; }
/** * Checks if a given block is less than needed size to meet our goal. * * @param blockSize - block len * @param item - Work item * @return true if this block meets our criteria, false otherwise. */ private boolean isLessThanNeeded(long blockSize, DiskBalancerWorkItem item) { long bytesToCopy = item.getBytesToCopy() - item.getBytesCopied(); bytesToCopy = bytesToCopy + ((bytesToCopy * getBlockTolerancePercentage(item)) / 100); return (blockSize <= bytesToCopy) ? true : false; }