/** * dumps the contents of recentInvalidateSets */ private synchronized void dumpRecentInvalidateSets(PrintWriter out) { int size = recentInvalidateSets.values().size(); out.println("Metasave: Blocks waiting deletion from "+size+" datanodes."); if (size == 0) { return; } for(Map.Entry<String,Collection<Block>> entry : recentInvalidateSets.entrySet()) { Collection<Block> blocks = entry.getValue(); if (blocks.size() > 0) { out.println(datanodeMap.get(entry.getKey()).getName() + blocks); } } }
/** * Stop decommissioning the specified datanodes. */ public void stopDecommission (DatanodeDescriptor node) throws IOException { LOG.info("Stop Decommissioning node " + node.getName()); node.stopDecommission(); }
/** * dumps the contents of recentInvalidateSets */ private void dumpRecentInvalidateSets(PrintWriter out) { int size = recentInvalidateSets.values().size(); out.println("Metasave: Blocks " + pendingDeletionBlocksCount + " waiting deletion from " + size + " datanodes."); if (size == 0) { return; } for (Map.Entry<String, LightWeightHashSet<Block>> entry : recentInvalidateSets .entrySet()) { LightWeightHashSet<Block> blocks = entry.getValue(); if (blocks.size() > 0) { out.println(datanodeMap.get(entry.getKey()).getName() + blocks); } } }
synchronized private void handlePendingStopDecommission() { if (pendingToStopDecommission) { LOG.info("Stop (delayed) Decommissioning node " + nodeBeingCheck.getName()); nodeBeingCheck.stopDecommission(); nodeBeingCheck.decommissioningStatus.set(0, 0, 0); pendingToStopDecommission = false; } }
void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) { nodeDescr.resetBlocks(); removeFromInvalidates(nodeDescr); NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedRemoveDatanode: " + nodeDescr.getName() + " is out of service now."); }
/** * Adds block to list of blocks which will be invalidated on * all its datanodes. */ private void addToInvalidates(Block b, boolean ackRequired) { StringBuilder sb = new StringBuilder(); for (Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(b); it.hasNext();) { DatanodeDescriptor node = it.next(); addToInvalidatesNoLog(b, node, ackRequired); sb.append(node.getName()); sb.append(' '); } NameNode.stateChangeLog.info("BLOCK* NameSystem.addToInvalidates: " + b.getBlockName() + " is added to invalidSet of " + sb); }
public Integer findNode(DatanodeDescriptor node) { Integer retVal = rackNodesMap.get(node.getHostName()); if (retVal == null) { retVal = rackNodesMap.get(node.getName()); if (retVal == null) { retVal = rackNodesMap.get(node.getHost()); if (retVal == null) { LOG.info("Didn't find " + node.getHostName() + " - " + node.getName() + " - " + node.getHost()); } } } return retVal; } }
void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) { nodeDescr.resetBlocks(); removeFromInvalidates(nodeDescr.getStorageID()); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedRemoveDatanode: " + nodeDescr.getName() + " is out of service now."); } }
void unprotectedAddDatanode(DatanodeDescriptor nodeDescr) { /* To keep host2DataNodeMap consistent with datanodeMap, remove from host2DataNodeMap the datanodeDescriptor removed from datanodeMap before adding nodeDescr to host2DataNodeMap. */ host2DataNodeMap.remove( datanodeMap.put(nodeDescr.getStorageID(), nodeDescr)); host2DataNodeMap.add(nodeDescr); NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedAddDatanode: " + "node " + nodeDescr.getName() + " is added to datanodeMap."); }
void unprotectedAddDatanode(DatanodeDescriptor nodeDescr) { /* To keep host2DataNodeMap consistent with datanodeMap, remove from host2DataNodeMap the datanodeDescriptor removed from datanodeMap before adding nodeDescr to host2DataNodeMap. */ host2DataNodeMap.remove( datanodeMap.put(nodeDescr.getStorageID(), nodeDescr)); host2DataNodeMap.add(nodeDescr); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.unprotectedAddDatanode: " + "node " + nodeDescr.getName() + " is added to datanodeMap."); } }
/** * Mark the block belonging to datanode as corrupt. * * @param blk Block to be added to CorruptReplicasMap * @param dn DatanodeDescriptor which holds the corrupt replica * @return if the block gets added or not */ public boolean addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn) { Collection<DatanodeDescriptor> nodes = getNodes(blk); if (nodes == null) { nodes = new TreeSet<DatanodeDescriptor>(); corruptReplicasMap.put(blk, nodes); } boolean added = false; if (!nodes.contains(dn)) { added = nodes.add(dn); NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ blk.getBlockName() + " added as corrupt on " + dn.getName() + " by " + Server.getRemoteIp()); } else { NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ "duplicate requested for " + blk.getBlockName() + " to add as corrupt " + "on " + dn.getName() + " by " + Server.getRemoteIp()); } return added; }
/** * Log a rejection of an addStoredBlock RPC, invalidate the reported block */ private void rejectAddStoredBlock(Block block, DatanodeDescriptor node, String msg, boolean ignoreInfoLogs) { if ((!isInSafeMode()) && (!ignoreInfoLogs)) { NameNode.stateChangeLog.info("BLOCK* NameSystem.addStoredBlock: " + "addStoredBlock request received for " + block + " size " + block.getNumBytes() + " but was rejected and added to invalidSet of " + node.getName() + " : " + msg); } addToInvalidatesNoLog(block, node, false); }
/** * Get data node by storage ID. * * @param nodeID * @return DatanodeDescriptor or null if the node is not found. * @throws IOException */ public DatanodeDescriptor getDatanode(DatanodeID nodeID) throws IOException { UnregisteredDatanodeException e = null; DatanodeDescriptor node = datanodeMap.get(nodeID.getStorageID()); if (node == null) return null; if (!node.getName().equals(nodeID.getName())) { e = new UnregisteredDatanodeException(nodeID, node); NameNode.stateChangeLog.fatal("BLOCK* NameSystem.getDatanode: " + e.getLocalizedMessage()); throw e; } return node; }
/** * Get data node by storage ID. * * @param nodeID * @return DatanodeDescriptor or null if the node is not found. * @throws IOException */ public DatanodeDescriptor getDatanode(DatanodeID nodeID) throws IOException { UnregisteredDatanodeException e = null; DatanodeDescriptor node = datanodeMap.get(nodeID.getStorageID()); if (node == null) { return null; } if (!node.getName().equals(nodeID.getName())) { e = new UnregisteredDatanodeException(nodeID, node); NameNode.stateChangeLog.fatal("BLOCK* NameSystem.getDatanode: " + e.getLocalizedMessage()); throw e; } return node; }
/** * Mark the block belonging to datanode as corrupt. * * @param blk Block to be added to CorruptReplicasMap * @param dn DatanodeDescriptor which holds the corrupt replica */ public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn) { Collection<DatanodeDescriptor> nodes = getNodes(blk); if (nodes == null) { nodes = new TreeSet<DatanodeDescriptor>(); corruptReplicasMap.put(blk, nodes); } if (!nodes.contains(dn)) { nodes.add(dn); NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ blk.getBlockName() + " added as corrupt on " + dn.getName() + " by " + Server.getRemoteIp()); } else { NameNode.stateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ "duplicate requested for " + blk.getBlockName() + " to add as corrupt " + "on " + dn.getName() + " by " + Server.getRemoteIp()); } if (NameNode.getNameNodeMetrics() != null) { NameNode.getNameNodeMetrics().numBlocksCorrupted.set( corruptReplicasMap.size()); } }
/** * Start decommissioning the specified datanode. */ private void startDecommission (DatanodeDescriptor node) throws IOException { if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { LOG.info("Start Decommissioning node " + node.getName()); node.startDecommission(); // // all the blocks that reside on this node have to be // replicated. Iterator<Block> decommissionBlocks = node.getBlockIterator(); while(decommissionBlocks.hasNext()) { Block block = decommissionBlocks.next(); updateNeededReplications(block, -1, 0); } } }
/** * Change, if appropriate, the admin state of a datanode to * decommission completed. Return true if decommission is complete. */ boolean checkDecommissionStateInternal(DatanodeDescriptor node) { // // Check to see if all blocks in this decommissioned // node has reached their target replication factor. // if (node.isDecommissionInProgress()) { if (!isReplicationInProgress(node)) { node.setDecommissioned(); LOG.info("Decommission complete for node " + node.getName()); } } if (node.isDecommissioned()) { return true; } return false; }
/** * Stop decommissioning the specified datanodes. */ private void stopDecommission(DatanodeDescriptor node) throws IOException { if ((node.isDecommissionInProgress() && ((Monitor) dnthread.getRunnable()).stopDecommission(node)) || node.isDecommissioned()) { LOG.info("Stop Decommissioning node " + node.getName()); synchronized (heartbeats) { updateStats(node, false); node.stopDecommission(); updateStats(node, true); } } }
/** * Start decommissioning the specified datanode. */ private void startDecommission(DatanodeDescriptor node) throws IOException { if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { LOG.info("Start Decommissioning node " + node.getName() + " with " + node.numBlocks() + " blocks."); synchronized (heartbeats) { updateStats(node, false); node.startDecommission(); updateStats(node, true); } if (((Monitor) dnthread.getRunnable()).startDecommision(node)) { node.setStartTime(now()); } } else if (node.isDecommissionInProgress()) { if (((Monitor) dnthread.getRunnable()).startDecommision(node)) { node.setStartTime(now()); } } }