private void removeFromExcessReplicateMap(Block block, DatanodeDescriptor node){ // No need for lock until we start accepting requests from clients. assert (!nameNode.isRpcServerRunning() || hasWriteLock()); Collection<Block> excessBlocks = excessReplicateMap.get(node.getStorageID()); if (excessBlocks != null) { if (excessBlocks.remove(block)) { excessBlocksCount--; if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("BLOCK* NameSystem.removeFromExcessReplicateMap: " + block + " is removed from excessBlocks"); } if (excessBlocks.size() == 0) { excessReplicateMap.remove(node.getStorageID()); } } } }
void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) { nodeDescr.resetBlocks(); removeFromInvalidates(nodeDescr.getStorageID()); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedRemoveDatanode: " + nodeDescr.getName() + " is out of service now."); } }
void unprotectedAddDatanode(DatanodeDescriptor nodeDescr) { /* To keep host2DataNodeMap consistent with datanodeMap, remove from host2DataNodeMap the datanodeDescriptor removed from datanodeMap before adding nodeDescr to host2DataNodeMap. */ host2DataNodeMap.remove( datanodeMap.put(nodeDescr.getStorageID(), nodeDescr)); host2DataNodeMap.add(nodeDescr); NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedAddDatanode: " + "node " + nodeDescr.getName() + " is added to datanodeMap."); }
/** * Decide if a replica is valid * @param node datanode the block is located * @param block a block * @return true if a replica is valid */ private boolean isGoodReplica(DatanodeDescriptor node, Block block) { Collection<Block> excessBlocks = excessReplicateMap.get(node.getStorageID()); Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block); return (nodesCorrupt == null || !nodesCorrupt.contains(node)) // not corrupt // not over scheduling for replication && (node.getNumberOfBlocksToBeReplicated() < maxReplicationStreams) // not alredy scheduled for removal && (excessBlocks == null || !excessBlocks.contains(block)) && !node.isDecommissioned(); // not decommissioned }
void unprotectedAddDatanode(DatanodeDescriptor nodeDescr) { /* To keep host2DataNodeMap consistent with datanodeMap, remove from host2DataNodeMap the datanodeDescriptor removed from datanodeMap before adding nodeDescr to host2DataNodeMap. */ host2DataNodeMap.remove( datanodeMap.put(nodeDescr.getStorageID(), nodeDescr)); host2DataNodeMap.add(nodeDescr); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedAddDatanode: " + "node " + nodeDescr.getName() + " is added to datanodeMap."); } }
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { ArrayList<String> machineSet = new ArrayList<String>(blocksMap.numNodes(block)); for(Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); it.hasNext();) { String storageID = it.next().getStorageID(); // filter invalidate replicas Collection<Block> blocks = recentInvalidateSets.get(storageID); if(blocks==null || !blocks.contains(block)) { machineSet.add(storageID); } } if(machineSet.size() == 0) { return 0; } else { results.add(new BlockWithLocations(block, machineSet.toArray(new String[machineSet.size()]))); return block.getNumBytes(); } }
/** * Get all valid locations of the block & add the block to results * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List<BlockWithLocations> results) { ArrayList<String> machineSet = new ArrayList<String>(blocksMap.numNodes(block)); for (Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); it.hasNext();) { String storageID = it.next().getStorageID(); // filter invalidate replicas LightWeightHashSet<Block> blocks = recentInvalidateSets.get(storageID); if (blocks == null || !blocks.contains(block)) { machineSet.add(storageID); } } if (machineSet.size() == 0) { return 0; } else { results.add(new BlockWithLocations(block, machineSet.toArray(new String[machineSet.size()]))); return block.getNumBytes(); } }
StringBuilder sb = new StringBuilder(); for (DatanodeDescriptor datanode : this.targets) { sb.append(datanode.getName() + ":" + datanode.getStorageID() + " "); + " node=" + node.getName() + ":" + node.getStorageID() + " inode=" + this + " targets=" + sb);
excessReplicateMap.get(node.getStorageID()); if (blocksExcess != null && blocksExcess.contains(b)) { excess++;
/** * Find how many of the containing nodes are "extra", if any. * If there are any extras, call chooseExcessReplicates() to * mark them in the excessReplicateMap. */ private void processOverReplicatedBlock(Block block, short replication, DatanodeDescriptor addedNode, DatanodeDescriptor delNodeHint) { if(addedNode == delNodeHint) { delNodeHint = null; } Collection<DatanodeDescriptor> nonExcess = new ArrayList<DatanodeDescriptor>(); Collection<DatanodeDescriptor> corruptNodes = corruptReplicas.getNodes(block); for (Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block); it.hasNext();) { DatanodeDescriptor cur = it.next(); Collection<Block> excessBlocks = excessReplicateMap.get(cur.getStorageID()); if (excessBlocks == null || !excessBlocks.contains(block)) { if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) { // exclude corrupt replicas if (corruptNodes == null || !corruptNodes.contains(cur)) { nonExcess.add(cur); } } } } chooseExcessReplicates(nonExcess, block, replication, addedNode, delNodeHint); }
/** * Counts the number of nodes in the given list into active and * decommissioned counters. */ private NumberReplicas countNodes(Block b, Iterator<DatanodeDescriptor> nodeIter) { int count = 0; int live = 0; int corrupt = 0; int excess = 0; Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b); while (nodeIter.hasNext()) { DatanodeDescriptor node = nodeIter.next(); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) { corrupt++; } else if (node.isDecommissionInProgress() || node.isDecommissioned()) { count++; } else { Collection<Block> blocksExcess = excessReplicateMap.get(node.getStorageID()); if (blocksExcess != null && blocksExcess.contains(b)) { excess++; } else { live++; } } } return new NumberReplicas(live, count, corrupt, excess); }
Collection<Block> excessBlocks = excessReplicateMap.get(node.getStorageID()); if (excessBlocks != null) { excessBlocks.remove(block); +block+" is removed from excessBlocks"); if (excessBlocks.size() == 0) { excessReplicateMap.remove(node.getStorageID());
it.hasNext();) { DatanodeDescriptor cur = it.next(); Collection<Block> excessBlocks = excessReplicateMap.get(cur.getStorageID()); if (excessBlocks == null || !excessBlocks.contains(block)) { if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) {
DatanodeDescriptor node = it.next(); Collection<Block> excessBlocks = excessReplicateMap.get(node.getStorageID()); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) corrupt++;
DatanodeDescriptor node = it.next(); Collection<Block> excessBlocks = excessReplicateMap.get(node.getStorageID()); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) { corrupt++;
String datanodeId = dd.getStorageID(); LightWeightHashSet<Block> v = recentInvalidateSets.get(datanodeId); if (v != null && v.remove(last)) {
while (iter.hasNext()) { DatanodeDescriptor dn = iter.next(); Collection<Block> blocks = namesystem.blockManager.excessReplicateMap.get(dn.getStorageID()); if (blocks == null || !blocks.contains(block) ) { nonExcessDN = dn;