/** Send IBRs to namenode. */ void sendIBRs(DatanodeProtocol namenode, DatanodeRegistration registration, String bpid) throws IOException { // Generate a list of the pending reports for each storage under the lock final StorageReceivedDeletedBlocks[] reports = generateIBRs(); if (reports.length == 0) { // Nothing new to report. return; } // Send incremental block reports to the Namenode outside the lock if (LOG.isDebugEnabled()) { LOG.debug("call blockReceivedAndDeleted: " + Arrays.toString(reports)); } boolean success = false; final long startTime = monotonicNow(); try { namenode.blockReceivedAndDeleted(registration, bpid, reports); success = true; } finally { if (success) { dnMetrics.addIncrementalBlockReport(monotonicNow() - startTime); lastIBR = startTime; } else { // If we didn't succeed in sending the report, put all of the // blocks back onto our queue, but only in the case where we // didn't put something newer in the meantime. putMissing(reports); } } }
impl.blockReceivedAndDeleted(PBHelper.convert(request.getRegistration()), request.getBlockPoolId(), info); } catch (IOException e) {
/** Send IBRs to namenode. */ void sendIBRs(DatanodeProtocol namenode, DatanodeRegistration registration, String bpid, DataNodeMetrics metrics) throws IOException { // Generate a list of the pending reports for each storage under the lock final StorageReceivedDeletedBlocks[] reports = generateIBRs(); if (reports.length == 0) { // Nothing new to report. return; } // Send incremental block reports to the Namenode outside the lock if (LOG.isDebugEnabled()) { LOG.debug("call blockReceivedAndDeleted: " + Arrays.toString(reports)); } boolean success = false; final long startTime = monotonicNow(); try { namenode.blockReceivedAndDeleted(registration, bpid, reports); success = true; } finally { metrics.addIncrementalBlockReport(monotonicNow() - startTime); if (success) { lastIBR = startTime; } else { // If we didn't succeed in sending the report, put all of the // blocks back onto our queue, but only in the case where we // didn't put something newer in the meantime. putMissing(reports); } } }
@Override public BlockReceivedAndDeletedResponseProto blockReceivedAndDeleted( RpcController controller, BlockReceivedAndDeletedRequestProto request) throws ServiceException { List<StorageReceivedDeletedBlocksProto> sBlocks = request.getBlocksList(); StorageReceivedDeletedBlocks[] info = new StorageReceivedDeletedBlocks[sBlocks.size()]; for (int i = 0; i < sBlocks.size(); i++) { StorageReceivedDeletedBlocksProto sBlock = sBlocks.get(i); List<ReceivedDeletedBlockInfoProto> list = sBlock.getBlocksList(); ReceivedDeletedBlockInfo[] rdBlocks = new ReceivedDeletedBlockInfo[list.size()]; for (int j = 0; j < list.size(); j++) { rdBlocks[j] = PBHelper.convert(list.get(j)); } if (sBlock.hasStorage()) { info[i] = new StorageReceivedDeletedBlocks( PBHelper.convert(sBlock.getStorage()), rdBlocks); } else { info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks); } } try { impl.blockReceivedAndDeleted(PBHelper.convert(request.getRegistration()), request.getBlockPoolId(), info); } catch (IOException e) { throw new ServiceException(e); } return VOID_BLOCK_RECEIVED_AND_DELETE_RESPONSE; }
@Override public BlockReceivedAndDeletedResponseProto blockReceivedAndDeleted( RpcController controller, BlockReceivedAndDeletedRequestProto request) throws ServiceException { List<StorageReceivedDeletedBlocksProto> sBlocks = request.getBlocksList(); StorageReceivedDeletedBlocks[] info = new StorageReceivedDeletedBlocks[sBlocks.size()]; for (int i = 0; i < sBlocks.size(); i++) { StorageReceivedDeletedBlocksProto sBlock = sBlocks.get(i); List<ReceivedDeletedBlockInfoProto> list = sBlock.getBlocksList(); ReceivedDeletedBlockInfo[] rdBlocks = new ReceivedDeletedBlockInfo[list.size()]; for (int j = 0; j < list.size(); j++) { rdBlocks[j] = PBHelper.convert(list.get(j)); } if (sBlock.hasStorage()) { info[i] = new StorageReceivedDeletedBlocks( PBHelper.convert(sBlock.getStorage()), rdBlocks); } else { info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks); } } try { impl.blockReceivedAndDeleted(PBHelper.convert(request.getRegistration()), request.getBlockPoolId(), info); } catch (IOException e) { throw new ServiceException(e); } return VOID_BLOCK_RECEIVED_AND_DELETE_RESPONSE; }
+ receivedAndDeletedBlockArray.length + " blocks to " + nnAddr); nsNamenode.blockReceivedAndDeleted(nsRegistration, receivedAndDeletedBlockArray); if (LOG.isDebugEnabled()) { LOG.debug("finshed blockReceivedAndDeleted to " + nnAddr
/** Send IBRs to namenode. */ void sendIBRs(DatanodeProtocol namenode, DatanodeRegistration registration, String bpid, DataNodeMetrics metrics) throws IOException { // Generate a list of the pending reports for each storage under the lock final StorageReceivedDeletedBlocks[] reports = generateIBRs(); if (reports.length == 0) { // Nothing new to report. return; } // Send incremental block reports to the Namenode outside the lock if (LOG.isDebugEnabled()) { LOG.debug("call blockReceivedAndDeleted: " + Arrays.toString(reports)); } boolean success = false; final long startTime = monotonicNow(); try { namenode.blockReceivedAndDeleted(registration, bpid, reports); success = true; } finally { metrics.addIncrementalBlockReport(monotonicNow() - startTime); if (success) { lastIBR = startTime; } else { // If we didn't succeed in sending the report, put all of the // blocks back onto our queue, but only in the case where we // didn't put something newer in the meantime. putMissing(reports); } } }
private ExtendedBlock addBlocks(String fileName, String clientName) throws IOException { ExtendedBlock prevBlock = null; for(int jdx = 0; jdx < blocksPerFile; jdx++) { LocatedBlock loc = addBlock(fileName, clientName, prevBlock, null, INodeId.GRANDFATHER_INODE_ID, null); prevBlock = loc.getBlock(); for(DatanodeInfo dnInfo : loc.getLocations()) { int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getXferAddr()); datanodes[dnIdx].addBlock(loc.getBlock().getLocalBlock()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( loc.getBlock().getLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( datanodes[dnIdx].storage.getStorageID(), rdBlocks) }; dataNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, bpid, report); } // IBRs are asynchronously processed by NameNode. The next // ClientProtocol#addBlock() may throw NotReplicatedYetException. } return prevBlock; }
/** * Transfer blocks to another data-node. * Just report on behalf of the other data-node * that the blocks have been received. */ private int transferBlocks( Block blocks[], DatanodeInfo xferTargets[][], String targetStorageIDs[][] ) throws IOException { for(int i = 0; i < blocks.length; i++) { DatanodeInfo blockTargets[] = xferTargets[i]; for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; String targetStorageID = targetStorageIDs[i][t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( targetStorageID, rdBlocks) }; dataNodeProto.blockReceivedAndDeleted(receivedDNReg, bpid, report); } } return blocks.length; } }
dnp.blockReceivedAndDeleted(reg, poolId, storageBlocks); BlockManager bm = cluster.getNamesystem().getBlockManager();