final DDatanode target = task.target.getDDatanode(); final PendingMove pendingBlock = new PendingMove(this, task.target); if (target.addPendingBlock(pendingBlock)) { } else { target.removePendingBlock(pendingBlock);
for(DatanodeStorageReport r : reports) { final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo()); final boolean isSource = Util.isIncluded(sourceNodes, dn.getDatanodeInfo()); for(StorageType t : StorageType.getMovableTypes()) { final Double utilization = policy.getUtilization(r, t); final Source s = dn.addSource(t, maxSize2Move, dispatcher); if (thresholdDiff <= 0) { // within threshold aboveAvgUtilized.add(s); g = dn.addTarget(t, maxSize2Move); if (thresholdDiff <= 0) { // within threshold belowAvgUtilized.add(g);
public void executePendingMove(final PendingMove p) { // move the reportedBlock final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
} catch (IOException e) { LOG.warn("Failed to move " + this + ": " + e.getMessage()); target.getDDatanode().setHasFailure(); IOUtils.closeSocket(sock); proxySource.removePendingBlock(this); target.getDDatanode().removePendingBlock(this);
} catch (IOException e) { LOG.warn("Failed to move " + this + ": " + e.getMessage()); target.getDDatanode().setHasFailure(); IOUtils.closeSocket(sock); proxySource.removePendingBlock(this); target.getDDatanode().removePendingBlock(this);
final DDatanode target = task.target.getDDatanode(); final PendingMove pendingBlock = new PendingMove(this, task.target); if (target.addPendingBlock(pendingBlock)) { } else { target.removePendingBlock(pendingBlock);
final DDatanode target = task.target.getDDatanode(); final PendingMove pendingBlock = new PendingMove(this, task.target); if (target.addPendingBlock(pendingBlock)) { } else { target.removePendingBlock(pendingBlock);
/** * Wait for all block move confirmations. * @return true if there is failed move execution */ public static boolean waitForMoveCompletion( Iterable<? extends StorageGroup> targets) { boolean hasFailure = false; for(;;) { boolean empty = true; for (StorageGroup t : targets) { if (!t.getDDatanode().isPendingQEmpty()) { empty = false; break; } else { hasFailure |= t.getDDatanode().hasFailure; } } if (empty) { return hasFailure; // all pending queues are empty } try { Thread.sleep(1000); } catch (InterruptedException ignored) { } } }
/** * Wait for all block move confirmations. * @return true if there is failed move execution */ public static boolean waitForMoveCompletion( Iterable<? extends StorageGroup> targets) { boolean hasFailure = false; for(;;) { boolean empty = true; for (StorageGroup t : targets) { if (!t.getDDatanode().isPendingQEmpty()) { empty = false; break; } else { hasFailure |= t.getDDatanode().hasFailure; } } if (empty) { return hasFailure; // all pending queues are empty } try { Thread.sleep(1000); } catch (InterruptedException ignored) { } } }
public void executePendingMove(final PendingMove p) { // move the block final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
public void executePendingMove(final PendingMove p) { // move the block final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
void init() throws IOException { initStoragePolicies(); final List<DatanodeStorageReport> reports = dispatcher.init(); for(DatanodeStorageReport r : reports) { final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo()); for(StorageType t : StorageType.getMovableTypes()) { final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher); final long maxRemaining = getMaxRemaining(r, t); final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t, maxRemaining) : null; storages.add(source, target); } } }
public DDatanode newDatanode(DatanodeInfo datanode) { return new DDatanode(datanode, maxConcurrentMovesPerNode); }
void init() throws IOException { initStoragePolicies(); final List<DatanodeStorageReport> reports = dispatcher.init(); for(DatanodeStorageReport r : reports) { final DDatanode dn = dispatcher.newDatanode(r.getDatanodeInfo()); for(StorageType t : StorageType.getMovableTypes()) { final Source source = dn.addSource(t, Long.MAX_VALUE, dispatcher); final long maxRemaining = getMaxRemaining(r, t); final StorageGroup target = maxRemaining > 0L ? dn.addTarget(t, maxRemaining) : null; storages.add(source, target); } } }
public DDatanode newDatanode(DatanodeInfo datanode) { return new DDatanode(datanode, maxConcurrentMovesPerNode); }
/** Add a scheduled block move to the node */ synchronized boolean addPendingBlock(PendingMove pendingBlock) { if (!isDelayActive()) { return pendings.add(pendingBlock); } return false; }
/** Add a scheduled block move to the node */ synchronized boolean addPendingBlock(PendingMove pendingBlock) { if (!isDelayActive()) { return pendings.add(pendingBlock); } return false; }
public StorageGroup addTarget(StorageType storageType, long maxSize2Move) { final StorageGroup g = new StorageGroup(storageType, maxSize2Move); put(storageType, g, targetMap); return g; }