String toSnapshot = getSnapshotName(argv[2]); try { SnapshotDiffReport diffReport = dfs.getSnapshotDiffReport(snapshotRoot, fromSnapshot, toSnapshot); System.out.println(diffReport.toString());
@Override public SnapshotDiffReport next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem)fs; myDfs.getSnapshotDiffReport(p, fromSnapshot, toSnapshot); } else { throw new UnsupportedOperationException("Cannot perform snapshot" + " operations on a symlink to a non-DistributedFileSystem: " + snapshotDir + " -> " + p); } return null; } }.resolve(this, absF);
@Override public SnapshotDiffReport next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem)fs; myDfs.getSnapshotDiffReport(p, fromSnapshot, toSnapshot); } else { throw new UnsupportedOperationException("Cannot perform snapshot" + " operations on a symlink to a non-DistributedFileSystem: " + snapshotDir + " -> " + p); } return null; } }.resolve(this, absF);
@Override public SnapshotDiffReport next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem)fs; myDfs.getSnapshotDiffReport(p, fromSnapshot, toSnapshot); } else { throw new UnsupportedOperationException("Cannot perform snapshot" + " operations on a symlink to a non-DistributedFileSystem: " + snapshotDir + " -> " + p); } return null; } }.resolve(this, absF);
/** * Executes the filesystem operation. * @param fs filesystem instance to use. * @return A serialized JSON string of snapshot diffs. * @throws IOException thrown if an IO error occurred. */ @Override public String execute(FileSystem fs) throws IOException { SnapshotDiffReport sdr = null; if (fs instanceof DistributedFileSystem) { DistributedFileSystem dfs = (DistributedFileSystem) fs; sdr = dfs.getSnapshotDiffReport(path, oldSnapshotName, snapshotName); } else { throw new UnsupportedOperationException("getSnapshotDiff is not " + "supported for HttpFs on " + fs.getClass() + ". Please check your fs.defaultFS configuration"); } if (sdr != null) { return JsonUtil.toJsonString(sdr); } else { return ""; } } }
/** * Compute the snapshot diff on the given file system. Return true if the diff * is empty, i.e., no changes have happened in the FS. */ private static boolean checkNoChange(DistCpOptions inputOptions, DistributedFileSystem fs, Path path) { try { SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, inputOptions.getFromSnapshot(), ""); if (!targetDiff.getDiffList().isEmpty()) { DistCp.LOG.warn("The target has been modified since snapshot " + inputOptions.getFromSnapshot()); return false; } else { return true; } } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + path, e); } return false; }
@VisibleForTesting static DiffInfo[] getDiffs(DistCpOptions inputOptions, DistributedFileSystem fs, Path sourceDir, Path targetDir) { try { final String from = getSnapshotName(inputOptions.getFromSnapshot()); final String to = getSnapshotName(inputOptions.getToSnapshot()); SnapshotDiffReport sourceDiff = fs.getSnapshotDiffReport(sourceDir, from, to); return DiffInfo.getDiffs(sourceDiff, targetDir); } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + sourceDir, e); } return null; }
/** * Compute the snapshot diff on the given file system. Return true if the diff * is empty, i.e., no changes have happened in the FS. */ private boolean checkNoChange(DistributedFileSystem fs, Path path) { try { final String from = getSnapshotName(context.getFromSnapshot()); SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, from, ""); if (!targetDiff.getDiffList().isEmpty()) { DistCp.LOG.warn("The target has been modified since snapshot " + context.getFromSnapshot()); return false; } else { return true; } } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + path + " at snapshot " + context.getFromSnapshot(), e); } return false; }
@Test public void testSync2() throws Exception { initData2(source); initData2(target); enableAndCreateFirstSnapshot(); // make changes under source changeData2(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
@Test (timeout=60000) public void testRenameDirectoryInSnapshot() throws Exception { final Path sub2 = new Path(sub1, "sub2"); final Path sub3 = new Path(sub1, "sub3"); final Path sub2file1 = new Path(sub2, "sub2file1"); final String sub1snap1 = "sub1snap1"; hdfs.mkdirs(sub1); hdfs.mkdirs(sub2); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPL, SEED); SnapshotTestHelper.createSnapshot(hdfs, sub1, sub1snap1); // First rename the sub-directory. hdfs.rename(sub2, sub3); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, sub1snap1, ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertEquals(2, entries.size()); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), sub3.getName())); }
@Test public void testSync2() throws Exception { if (isSrcNotSameAsTgt) { initData2(source); } initData2(target); enableAndCreateFirstSnapshot(); // make changes under target changeData2(target); createSecondSnapshotAtTarget(); SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1"); System.out.println(report); syncAndVerify(); }
/** * Test a case where there are multiple source files with the same name. */ @Test public void testSync3() throws Exception { initData3(source); initData3(target); enableAndCreateFirstSnapshot(); // make changes under source changeData3(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
/** * Test a case where multiple level dirs are renamed. */ @Test public void testSync4() throws Exception { initData4(source); initData4(target); enableAndCreateFirstSnapshot(); // make changes under source changeData4(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
/** * Test a case where there are multiple source files with the same name. */ @Test public void testSync3() throws Exception { if (isSrcNotSameAsTgt) { initData3(source); } initData3(target); enableAndCreateFirstSnapshot(); // make changes under target changeData3(target); createSecondSnapshotAtTarget(); SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1"); System.out.println(report); syncAndVerify(); }
/** * Test a case with different delete and rename sequences. */ @Test public void testSync5() throws Exception { initData5(source); initData5(target); enableAndCreateFirstSnapshot(); // make changes under source changeData5(source); dfs.createSnapshot(source, "s2"); SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); syncAndVerify(); }
/** * Test a case with different delete and rename sequences. */ @Test public void testSync5() throws Exception { if (isSrcNotSameAsTgt) { initData5(source); } initData5(target); enableAndCreateFirstSnapshot(); // make changes under target int numDeletedAndModified = changeData5(target); createSecondSnapshotAtTarget(); SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1"); System.out.println(report); testAndVerify(numDeletedAndModified); }
/** check the correctness of the diff reports */ private void verifyDiffReport(Path dir, String from, String to, DiffReportEntry... entries) throws IOException { SnapshotDiffReport report = hdfs.getSnapshotDiffReport(dir, from, to); // reverse the order of from and to SnapshotDiffReport inverseReport = hdfs .getSnapshotDiffReport(dir, to, from); System.out.println(report.toString()); System.out.println(inverseReport.toString() + "\n"); assertEquals(entries.length, report.getDiffList().size()); assertEquals(entries.length, inverseReport.getDiffList().size()); for (DiffReportEntry entry : entries) { if (entry.getType() == DiffType.MODIFY) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains(entry)); } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); } } }
/** * Rename a file under a snapshottable directory, file exists * in a snapshot. */ @Test public void testRenameFileInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.createSnapshot(sub1, snap1); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); System.out.println("DiffList is " + diffReport.toString()); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), file2.getName())); }
/** * Test a case where multiple level dirs are renamed. */ @Test public void testSync4() throws Exception { if (isSrcNotSameAsTgt) { initData4(source); } initData4(target); enableAndCreateFirstSnapshot(); final FsShell shell = new FsShell(conf); lsr("Before change target: ", shell, target); // make changes under target int numDeletedAndModified = changeData4(target); createSecondSnapshotAtTarget(); SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1"); System.out.println(report); testAndVerify(numDeletedAndModified); }
/** * Rename a file under a snapshottable directory, file does not exist * in a snapshot. */ @Test (timeout=60000) public void testRenameFileNotInSnapshot() throws Exception { hdfs.mkdirs(sub1); hdfs.allowSnapshot(sub1); hdfs.createSnapshot(sub1, snap1); DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPL, SEED); hdfs.rename(file1, file2); // Query the diff report and make sure it looks as expected. SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List<DiffReportEntry> entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), null)); }