private static INodeFile newINodeFile( long id, PermissionStatus permissions, long mtime, long atime, Short replication, Byte ecPolicyID, long preferredBlockSize, byte storagePolicyId, BlockType blockType) { return new INodeFile(id, null, permissions, mtime, atime, BlockInfo.EMPTY_ARRAY, replication, ecPolicyID, preferredBlockSize, storagePolicyId, blockType); }
Preconditions.checkArgument(newNode.isUnderConstruction()); PermissionStatus permissions = newNode.getPermissionStatus(); AddOp op = AddOp.getInstance(cache.get()) .setInodeId(newNode.getId()) .setPath(path) .setReplication(newNode.getFileReplication()) .setModificationTime(newNode.getModificationTime()) .setAccessTime(newNode.getAccessTime()) .setBlockSize(newNode.getPreferredBlockSize()) .setBlocks(newNode.getBlocks()) .setPermissionStatus(permissions) .setClientName(newNode.getFileUnderConstructionFeature().getClientName()) .setClientMachine( newNode.getFileUnderConstructionFeature().getClientMachine()) .setOverwrite(overwrite) .setStoragePolicyId(newNode.getLocalStoragePolicyID()) .setErasureCodingPolicyId(newNode.getErasureCodingPolicyID()); AclFeature f = newNode.getAclFeature(); if (f != null) { op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode)); XAttrFeature x = newNode.getXAttrFeature(); if (x != null) { op.setXAttrs(x.getXAttrs());
String path = cons.getLocalName(); INodeFile oldnode = null; boolean inSnapshot = false; oldnode = namesystem.dir.getInode(cons.getId()).asFile(); inSnapshot = true; } else { path = renameReservedPathsOnUpgrade(path, getLayoutVersion()); final INodesInPath iip = fsDir.getINodesInPath(path, DirOp.WRITE); oldnode = INodeFile.valueOf(iip.getLastINode(), path); FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature(); oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine()); if (oldnode.numBlocks() > 0) { BlockInfo ucBlock = cons.getLastBlock(); oldnode.setBlock(oldnode.numBlocks() - 1, info); namesystem.leaseManager.addLease(uc.getClientName(), oldnode.getId());
private void updateRemovedUnderConstructionFiles( ReclaimContext reclaimContext) { if (isUnderConstruction() && reclaimContext.removedUCFiles != null) { reclaimContext.removedUCFiles.add(getId()); } }
/** * Serialize a {@link INodeFile} node * @param file The INodeFile to write * @param out The {@link DataOutputStream} where the fields are written * @param writeUnderConstruction Whether to write under construction information */ public static void writeINodeFile(INodeFile file, DataOutput out, boolean writeUnderConstruction) throws IOException { writeLocalName(file, out); out.writeLong(file.getId()); out.writeShort(file.getFileReplication()); out.writeLong(file.getModificationTime()); out.writeLong(file.getAccessTime()); out.writeLong(file.getPreferredBlockSize()); writeBlocks(file.getBlocks(), out); SnapshotFSImageFormat.saveFileDiffList(file, out); if (writeUnderConstruction) { if (file.isUnderConstruction()) { out.writeBoolean(true); final FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature(); writeString(uc.getClientName(), out); writeString(uc.getClientMachine(), out); } else { out.writeBoolean(false); } } writePermissionStatus(file, out); }
static void writeINodeUnderConstruction(DataOutputStream out, INodeFile cons, String path) throws IOException { writeString(path, out); out.writeLong(cons.getId()); out.writeShort(cons.getFileReplication()); out.writeLong(cons.getModificationTime()); out.writeLong(cons.getPreferredBlockSize()); writeBlocks(cons.getBlocks(), out); cons.getPermissionStatus().write(out); FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature(); writeString(uc.getClientName(), out); writeString(uc.getClientMachine(), out); out.writeInt(0); // do not store locations of last block }
/** * Add close lease record to edit log. */ public void logCloseFile(String path, INodeFile newNode) { CloseOp op = CloseOp.getInstance(cache.get()) .setPath(path) .setReplication(newNode.getFileReplication()) .setModificationTime(newNode.getModificationTime()) .setAccessTime(newNode.getAccessTime()) .setBlockSize(newNode.getPreferredBlockSize()) .setBlocks(newNode.getBlocks()) .setPermissionStatus(newNode.getPermissionStatus()); logEdit(op); }
src = iFile.getFullPathName(); if (isFileDeleted(iFile)) { throw new FileNotFoundException("File not found: " + src + ", likely due to delayed block removal"); if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) && iFile.getLastBlock().isComplete()) { if (LOG.isDebugEnabled()) { LOG.debug("Unexpected block (=" + oldBlock + ") since the file (=" + iFile.getLocalName() + ") is not under construction"); (BlockInfoContiguousUnderConstruction) iFile.getLastBlock(); long recoveryId = truncatedBlock.getBlockRecoveryId(); boolean copyTruncate = boolean remove = iFile.removeLastBlock(blockToDel); if (remove) { blockManager.removeBlock(storedBlock); iFile.setLastBlock(truncatedBlock, trimmedStorageInfos); } else { iFile.setLastBlock(storedBlock, trimmedStorageInfos); if (closeFile) { blockManager.markBlockReplicasAsCorrupt(storedBlock, if(copyTruncate) { closeFileCommitBlocks(src, iFile, truncatedBlock); if(!iFile.isBlockInLatestSnapshot(storedBlock)) {
/** * Add close lease record to edit log. */ public void logCloseFile(String path, INodeFile newNode) { CloseOp op = CloseOp.getInstance(); op.set(path, newNode.getReplication(), newNode.getModificationTime(), newNode.getAccessTime(), newNode.getPreferredBlockSize(), newNode.getBlocks(), newNode.getPermissionStatus(), null, null); logEdit(op); }
try { final INodeFile fileINode = inodesInPath.getLastINode().asFile(); Preconditions.checkState(fileINode.isUnderConstruction()); fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(), numLocations, true); blockInfo = new BlockInfoStriped(block, ecPolicy); } else { fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(), fileINode.getFileReplication(), true); short numLocations = fileINode.getFileReplication(); blockInfo = new BlockInfoContiguous(block, numLocations); blockInfo.convertToBlockUnderConstruction( fileINode.addBlock(blockInfo);
parent.getLoaderContext().getStringTable()); final INodeFile file = new INodeFile(n.getId(), n.getName().toByteArray(), permissions, f.getModificationTime(), f.getAccessTime(), blocks, replication, ecPolicyID, int[] entries = AclEntryStatusFormat.toInt(loadAclEntries( f.getAcl(), state.getStringTable())); file.addAclFeature(new AclFeature(entries)); file.addXAttrFeature(new XAttrFeature( loadXAttrs(f.getXAttrs(), state.getStringTable()))); file.toUnderConstruction(uc.getClientName(), uc.getClientMachine()); fsn.leaseManager.addLease(uc.getClientName(), file.getId()); if (blocks.length > 0) { BlockInfo lastBlk = file.getLastBlock(); file.setBlock(file.numBlocks() - 1, ucBlk);
src = iFile.getFullPathName(); if (isFileDeleted(iFile)) { throw new FileNotFoundException("File not found: " + src + ", likely due to delayed block removal"); if ((!iFile.isUnderConstruction() || storedBlock.isComplete()) && iFile.getLastBlock().isComplete()) { if (LOG.isDebugEnabled()) { LOG.debug("Unexpected block (=" + oldBlock + ") since the file (=" + iFile.getLocalName() + ") is not under construction"); truncatedBlock = iFile.getLastBlock(); final long recoveryId = truncatedBlock.getUnderConstructionFeature() .getBlockRecoveryId(); boolean remove = iFile.removeLastBlock(blockToDel) != null; if (remove) { blockManager.removeBlock(storedBlock); iFile.convertLastBlockToUC(truncatedBlock, dsInfos); } else { iFile.convertLastBlockToUC(storedBlock, dsInfos); if (closeFile) { blockManager.markBlockReplicasAsCorrupt(oldBlock.getLocalBlock(), if(copyTruncate) { closeFileCommitBlocks(src, iFile, truncatedBlock); if(!iFile.isBlockInLatestSnapshot(storedBlock)) {
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path, true); if (oldFile != null && addCloseOp.overwrite) { assert newFile != null; iip = INodesInPath.replace(iip, iip.length() - 1, newFile); fsNamesys.leaseManager.addLease(addCloseOp.clientName, newFile.getId()); if (!oldFile.isUnderConstruction()) { newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID, false); newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID); ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( final INodeFile file = INodeFile.valueOf(iip.getLastINode(), path); file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID, false); file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID); ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( if (!file.isUnderConstruction() && logVersion <= LayoutVersion.BUGFIX_HDFS_2991_VERSION) { if (file.isUnderConstruction()) { fsNamesys.getLeaseManager().removeLease(file.getId()); file.toCompleteFile(file.getModificationTime(), 0, fsNamesys.getBlockManager().getMinReplication());
INodesInPath iip, INodeFile file) throws IOException { BlockInfoContiguous[] oldBlocks = file.getBlocks(); Block[] newBlocks = op.getBlocks(); String path = op.getPath(); if (!file.isUnderConstruction()) { throw new IOException("Trying to remove a block from file " + path + " which is not under construction."); newBlock, file.getBlockReplication()); } else { newBI = new BlockInfoContiguous(newBlock, file.getBlockReplication()); file.addBlock(newBI); fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ); src = iip.getPath(); final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src); if (fsd.isPermissionEnabled()) { fsd.checkPathAccess(pc, iip, FsAction.READ); ? inode.computeFileSize(iip.getPathSnapshotId()) : inode.computeFileSizeNotIncludingLastUcBlock(); boolean isUc = inode.isUnderConstruction(); if (iip.isSnapshot()) { inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo, ecPolicy); boolean updateAccessTime = fsd.isAccessTimeSupported() && !iip.isSnapshot() && now > inode.getAccessTime() + fsd.getAccessTimePrecision(); return new GetBlockLocationsResult(updateAccessTime, blocks); } finally {
final INodeFile temp = TestSnapshotBlocksMap.assertBlockCollection( tempFile.toString(), 1, fsdir, blockmanager); BlockInfoContiguous[] blocks = temp.getBlocks(); hdfs.delete(tempDir, true); assertEquals(metaChangeFile2.getName(), metaChangeFile2SCopy.getLocalName()); assertTrue(metaChangeFile2SCopy.isWithSnapshot()); assertFalse(metaChangeFile2SCopy.isUnderConstruction()); TestSnapshotBlocksMap.assertBlockCollection(new Path(snapshotNoChangeDir, metaChangeFile2SCopy.getLocalName()).toString(), 1, fsdir, blockmanager); metaChangeFile2SCopy.getFileReplication(Snapshot.CURRENT_STATE_ID)); assertEquals(REPLICATION_1, metaChangeFile2SCopy.getFileReplication(snapshot1.getId())); assertEquals(REPLICATION, metaChangeFile2SCopy.getFileReplication(snapshot0.getId())); final INodeFile newFileNode = TestSnapshotBlocksMap.assertBlockCollection( newFile.toString(), 1, fsdir, blockmanager); blocks = newFileNode.getBlocks(); checkQuotaUsageComputation(dir, 10L, BLOCKSIZE * REPLICATION * 5); hdfs.delete(sub, true); assertTrue(metaChangeFile1SCopy.isWithSnapshot()); assertFalse(metaChangeFile1SCopy.isUnderConstruction()); assertEquals(REPLICATION_1, metaChangeFile1SCopy.getFileReplication(Snapshot.CURRENT_STATE_ID)); assertEquals(REPLICATION_1, metaChangeFile1SCopy.getFileReplication(snapshot1.getId()));
throws IOException { BlockInfo[] oldBlocks = file.getBlocks(); Block[] newBlocks = op.getBlocks(); String path = op.getPath(); if (!file.isUnderConstruction()) { throw new IOException("Trying to remove a block from file " + path + " which is not under construction."); } else { newBI = new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication()); } else { newBI = new BlockInfoContiguous(newBlock, file.getFileReplication()); file.addBlock(newBI); fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
/** * Add a block to the file. Returns a reference to the added block. */ Block addBlock(String path, INode[] inodes, Block block) throws IOException { waitForReady(); synchronized (rootDir) { INodeFile fileNode = (INodeFile) inodes[inodes.length-1]; // check quota limits and updated space consumed updateCount(inodes, inodes.length-1, 0, fileNode.getPreferredBlockSize()*fileNode.getReplication()); // associate the new list of blocks with this file namesystem.blocksMap.addINode(block, fileNode); BlockInfo blockInfo = namesystem.blocksMap.getStoredBlock(block); fileNode.addBlock(blockInfo); NameNode.stateChangeLog.debug("DIR* FSDirectory.addFile: " + path + " with " + block + " block is added to the in-memory " + "file system"); } return block; }