/** * After a block becomes finalized, a datanode increases metric counter, * notifies namenode, and adds it to the block scanner * @param block block to close * @param delHint hint on which excess block to delete * @param storageUuid UUID of the storage where block is stored */ void closeBlock(ExtendedBlock block, String delHint, String storageUuid, boolean isTransientStorage) { metrics.incrBlocksWritten(); notifyNamenodeReceivedBlock(block, delHint, storageUuid, isTransientStorage); }
/** * Update replica with the new generation stamp and length. */ @Override // InterDatanodeProtocol public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important // for HA, since otherwise the standby node may lose track of the // block locations until the next block report. ExtendedBlock newBlock = new ExtendedBlock(oldBlock); newBlock.setGenerationStamp(recoveryId); newBlock.setBlockId(newBlockId); newBlock.setNumBytes(newLength); final String storageID = r.getStorageUuid(); notifyNamenodeReceivedBlock(newBlock, null, storageID, r.isOnTransientStorage()); return storageID; }
/** * Cleanup the old replica and notifies the NN about new replica. * * @param replicaInfo - Old replica to be deleted * @param newReplicaInfo - New replica object * @param bpid - block pool id */ private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas cleanupReplica(bpid, replicaInfo); // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
datanode.notifyNamenodeReceivedBlock( block, delHint, r.getStorageUuid(), r.isOnTransientStorage());
/** * After a block becomes finalized, a datanode increases metric counter, * notifies namenode, and adds it to the block scanner * @param block block to close * @param delHint hint on which excess block to delete * @param storageUuid UUID of the storage where block is stored */ void closeBlock(ExtendedBlock block, String delHint, String storageUuid, boolean isTransientStorage) { metrics.incrBlocksWritten(); notifyNamenodeReceivedBlock(block, delHint, storageUuid, isTransientStorage); }
/** * After a block becomes finalized, a datanode increases metric counter, * notifies namenode, and adds it to the block scanner * @param block block to close * @param delHint hint on which excess block to delete * @param storageUuid UUID of the storage where block is stored */ void closeBlock(ExtendedBlock block, String delHint, String storageUuid, boolean isTransientStorage) { metrics.incrBlocksWritten(); notifyNamenodeReceivedBlock(block, delHint, storageUuid, isTransientStorage); }
datanode.data.finalizeBlock(block); datanode.myMetrics.blocksWritten.inc(); datanode.notifyNamenodeReceivedBlock(block, DataNode.EMPTY_DEL_HINT); if (ClientTraceLog.isInfoEnabled() &&
/** {@inheritDoc} */ public void updateBlock(int namespaceId, Block oldblock, Block newblock, boolean finalize) throws IOException { LOG.info("namespaceId: " + namespaceId + ", oldblock=" + oldblock + "(length=" + oldblock.getNumBytes() + "), newblock=" + newblock + "(length=" + newblock.getNumBytes() + "), datanode=" + getDatanodeInfo()); data.updateBlock(namespaceId, oldblock, newblock); if (finalize) { data.finalizeBlockIfNeeded(namespaceId, newblock); myMetrics.blocksWritten.inc(); notifyNamenodeReceivedBlock(namespaceId, newblock, null); LOG.info("Received block " + newblock + " of size " + newblock.getNumBytes() + " as part of lease recovery."); } }
/** {@inheritDoc} */ public void updateBlock(Block oldblock, Block newblock, boolean finalize) throws IOException { LOG.info("oldblock=" + oldblock + "(length=" + oldblock.getNumBytes() + "), newblock=" + newblock + "(length=" + newblock.getNumBytes() + "), datanode=" + dnRegistration.getName()); data.updateBlock(oldblock, newblock); if (finalize) { data.finalizeBlockIfNeeded(newblock); myMetrics.incrBlocksWritten(); notifyNamenodeReceivedBlock(newblock, EMPTY_DEL_HINT); LOG.info("Received " + newblock + " of size " + newblock.getNumBytes() + " as part of lease recovery"); } }
/** {@inheritDoc} */ public void updateBlock(Block oldblock, Block newblock, boolean finalize) throws IOException { LOG.info("oldblock=" + oldblock + "(length=" + oldblock.getNumBytes() + "), newblock=" + newblock + "(length=" + newblock.getNumBytes() + "), datanode=" + dnRegistration.getName()); data.updateBlock(oldblock, newblock); if (finalize) { data.finalizeBlock(newblock); myMetrics.blocksWritten.inc(); notifyNamenodeReceivedBlock(newblock, EMPTY_DEL_HINT); LOG.info("Received block " + newblock + " of size " + newblock.getNumBytes() + " as part of lease recovery."); } }
datanode.notifyNamenodeReceivedBlock(namespaceId, block, sourceID);
datanode.notifyNamenodeReceivedBlock(block, sourceID);
datanode.data.finalizeBlock(namespaceId, block); datanode.myMetrics.blocksWritten.inc(); datanode.notifyNamenodeReceivedBlock(namespaceId, block, null); if (ClientTraceLog.isInfoEnabled() && receiver.clientName.length() > 0) {
private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, File blockFile, File metaFile, long blockFileUsed, long metaFileUsed, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas if (blockFile.delete() || !blockFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed); if (metaFile.delete() || !metaFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed); } } // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
private void removeOldReplica(ReplicaInfo replicaInfo, ReplicaInfo newReplicaInfo, File blockFile, File metaFile, long blockFileUsed, long metaFileUsed, final String bpid) { // Before deleting the files from old storage we must notify the // NN that the files are on the new storage. Else a blockReport from // the transient storage might cause the NN to think the blocks are lost. // Replicas must be evicted from client short-circuit caches, because the // storage will no longer be same, and thus will require validating // checksum. This also stops a client from holding file descriptors, // which would prevent the OS from reclaiming the memory. ExtendedBlock extendedBlock = new ExtendedBlock(bpid, newReplicaInfo); datanode.getShortCircuitRegistry().processBlockInvalidation( ExtendedBlockId.fromExtendedBlock(extendedBlock)); datanode.notifyNamenodeReceivedBlock( extendedBlock, null, newReplicaInfo.getStorageUuid(), newReplicaInfo.isOnTransientStorage()); // Remove the old replicas if (blockFile.delete() || !blockFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed); if (metaFile.delete() || !metaFile.exists()) { ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed); } } // If deletion failed then the directory scanner will cleanup the blocks // eventually. }
public Boolean call() throws Exception { try { if (crossDatanode) { data.copyBlockLocal(srcFileSystem, srcBlockFile, srcNamespaceId, srcBlock, dstNamespaceId, dstBlock); } else { data.copyBlockLocal(srcFileSystem, data.getBlockFile(srcNamespaceId, srcBlock), srcNamespaceId, srcBlock, dstNamespaceId, dstBlock); } dstBlock.setNumBytes(srcBlock.getNumBytes()); notifyNamenodeReceivedBlock(dstNamespaceId, dstBlock, null); blockScanner.addBlock(dstNamespaceId, dstBlock); } catch (Exception e) { LOG.warn("Local block copy for src : " + srcBlock.getBlockName() + ", dst : " + dstBlock.getBlockName() + " failed", e); throw e; } return true; } }
/** * Update replica with the new generation stamp and length. */ @Override // InterDatanodeProtocol public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important // for HA, since otherwise the standby node may lose track of the // block locations until the next block report. ExtendedBlock newBlock = new ExtendedBlock(oldBlock); newBlock.setGenerationStamp(recoveryId); newBlock.setBlockId(newBlockId); newBlock.setNumBytes(newLength); final String storageID = r.getStorageUuid(); notifyNamenodeReceivedBlock(newBlock, null, storageID, r.isOnTransientStorage()); return storageID; }
/** * Update replica with the new generation stamp and length. */ @Override // InterDatanodeProtocol public String updateReplicaUnderRecovery(final ExtendedBlock oldBlock, final long recoveryId, final long newBlockId, final long newLength) throws IOException { final Replica r = data.updateReplicaUnderRecovery(oldBlock, recoveryId, newBlockId, newLength); // Notify the namenode of the updated block info. This is important // for HA, since otherwise the standby node may lose track of the // block locations until the next block report. ExtendedBlock newBlock = new ExtendedBlock(oldBlock); newBlock.setGenerationStamp(recoveryId); newBlock.setBlockId(newBlockId); newBlock.setNumBytes(newLength); final String storageID = r.getStorageUuid(); notifyNamenodeReceivedBlock(newBlock, null, storageID, r.isOnTransientStorage()); return storageID; }
datanode.notifyNamenodeReceivedBlock( block, delHint, r.getStorageUuid(), r.isOnTransientStorage());
datanode.notifyNamenodeReceivedBlock( block, delHint, r.getStorageUuid(), r.isOnTransientStorage());