/** * Implement the delete(Path, boolean) in checksum * file system. */ @Override public boolean delete(Path f, boolean recursive) throws IOException{ FileStatus fstatus = null; try { fstatus = fs.getFileStatus(f); } catch(FileNotFoundException e) { return false; } if (fstatus.isDirectory()) { //this works since the crcs are in the same //directories and the files. so we just delete //everything in the underlying filesystem return fs.delete(f, recursive); } else { Path checkFile = getChecksumFile(f); if (fs.exists(checkFile)) { fs.delete(checkFile, true); } return fs.delete(f, true); } }
boolean run(Path p) throws IOException { boolean status = apply(p); if (status) { Path checkFile = getChecksumFile(p); if (fs.exists(checkFile)) { apply(checkFile); } } return status; } abstract boolean apply(Path p) throws IOException;
/** * Rename files/dirs */ @Override @SuppressWarnings("deprecation") public boolean rename(Path src, Path dst) throws IOException { if (fs.isDirectory(src)) { return fs.rename(src, dst); } else { if (fs.isDirectory(dst)) { dst = new Path(dst, src.getName()); } boolean value = fs.rename(src, dst); if (!value) return false; Path srcCheckFile = getChecksumFile(src); Path dstCheckFile = getChecksumFile(dst); if (fs.exists(srcCheckFile)) { //try to rename checksum value = fs.rename(srcCheckFile, dstCheckFile); } else if (fs.exists(dstCheckFile)) { // no src checksum, so remove dst checksum value = fs.delete(dstCheckFile, true); } return value; } }
dst = new Path(dst, src.getName()); dst = getChecksumFile(dst); if (localFs.exists(dst)) { //remove old local checksum file localFs.delete(dst, true); Path checksumFile = getChecksumFile(src); if (copyCrc && fs.exists(checksumFile)) { //copy checksum file fs.copyToLocalFile(checksumFile, dst);
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress, FsPermission permission) throws IOException { super(DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, fs.getBytesPerSum())); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, permission, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), permission, true, sumBufferSize, replication, blockSize, null); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }
blockSize, progress); Path checkFile = getChecksumFile(f); if (fs.exists(checkFile)) { fs.delete(checkFile, true);
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize) throws IOException { super( file, fs.getFileStatus(file).getReplication() ); this.datas = fs.getRawFileSystem().open(file, bufferSize); this.fs = fs; Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize); sums = fs.getRawFileSystem().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; sums.readFully(version); if (!Arrays.equals(version, CHECKSUM_VERSION)) throw new IOException("Not a checksum file: "+sumFile); this.bytesPerSum = sums.readInt(); set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4); } catch (IOException e) { // mincing the message is terrible, but java throws permission // exceptions as FNF because that's all the method signatures allow! if (!(e instanceof FileNotFoundException) || e.getMessage().endsWith(" (Permission denied)")) { LOG.warn("Problem opening checksum file: "+ file + ". Ignoring exception: " , e); } set(fs.verifyChecksum, null, 1, 0); } }
/** * Rename files/dirs */ public boolean rename(Path src, Path dst) throws IOException { if (fs.isDirectory(src)) { return fs.rename(src, dst); } else { boolean value = fs.rename(src, dst); if (!value) return false; Path checkFile = getChecksumFile(src); if (fs.exists(checkFile)) { //try to rename checksum if (fs.isDirectory(dst)) { value = fs.rename(checkFile, dst); } else { value = fs.rename(checkFile, getChecksumFile(dst)); } } return value; } }
public enum LsOption { Recursive, WithBlockSize };
public enum LsOption { Recursive, WithBlockSize };
boolean run(Path p) throws IOException { boolean status = apply(p); if (status) { Path checkFile = getChecksumFile(p); if (fs.exists(checkFile)) { apply(checkFile); } } return status; } abstract boolean apply(Path p) throws IOException;
boolean run(Path p) throws IOException { boolean status = apply(p); if (status) { Path checkFile = getChecksumFile(p); if (fs.exists(checkFile)) { apply(checkFile); } } return status; } abstract boolean apply(Path p) throws IOException;
boolean run(Path p) throws IOException { boolean status = apply(p); if (status) { Path checkFile = getChecksumFile(p); if (fs.exists(checkFile)) { apply(checkFile); } } return status; } abstract boolean apply(Path p) throws IOException;
boolean run(Path p) throws IOException { boolean status = apply(p); if (status) { Path checkFile = getChecksumFile(p); if (fs.exists(checkFile)) { apply(checkFile); } } return status; } abstract boolean apply(Path p) throws IOException;
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, int bytesPerChecksum, Progressable progress) throws IOException { super(new CRC32(), bytesPerChecksum, 4); this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerChecksum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, sumBufferSize, replication, blockSize); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerChecksum); }
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { super(new CRC32(), fs.getBytesPerSum(), 4); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, sumBufferSize, replication, blockSize); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }
/** * Set replication for an existing file. * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt> * @param src file name * @param replication new replication * @throws IOException * @return true if successful; * false if file does not exist or is a directory */ public boolean setReplication(Path src, short replication) throws IOException { boolean value = fs.setReplication(src, replication); if (!value) return false; Path checkFile = getChecksumFile(src); if (exists(checkFile)) fs.setReplication(checkFile, replication); return true; }
/** * Set replication for an existing file. * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt> * @param src file name * @param replication new replication * @throws IOException * @return true if successful; * false if file does not exist or is a directory */ public boolean setReplication(Path src, short replication) throws IOException { boolean value = fs.setReplication(src, replication); if (!value) return false; Path checkFile = getChecksumFile(src); if (exists(checkFile)) fs.setReplication(checkFile, replication); return true; }
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress, FsPermission permission) throws IOException { super(new PureJavaCrc32(), fs.getBytesPerSum(), 4); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, permission, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), permission, true, sumBufferSize, replication, blockSize, null); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress, FsPermission permission) throws IOException { super(DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, fs.getBytesPerSum())); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, permission, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), permission, true, sumBufferSize, replication, blockSize, null); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }