public void run(FileSystem fs) throws IOException { DFSTestUtil.appendFile(fs, file1, "new bytes"); }});
@Override public void run(FileSystem fs) throws IOException { DFSTestUtil.appendFile(fs, file1, "new bytes"); }});
@Override void modify() throws Exception { assertTrue(fs.exists(file)); DFSTestUtil.appendFile(fs, file, appendLen); }
private int changeData6(Path dir) throws Exception { final Path foo = new Path(dir, "foo"); final Path bar = new Path(dir, "bar"); final Path foo2 = new Path(dir, "foo2"); final Path foo_f1 = new Path(foo, "f1"); int numCreatedModified = 0; dfs.rename(foo, foo2); dfs.rename(bar, foo); dfs.rename(foo2, bar); DFSTestUtil.appendFile(dfs, foo_f1, (int) BLOCK_SIZE); numCreatedModified += 1; // modify ./bar/f1 return numCreatedModified; }
private int changeData6(Path dir) throws Exception { final Path foo = new Path(dir, "foo"); final Path bar = new Path(dir, "bar"); final Path foo2 = new Path(dir, "foo2"); final Path foo_f1 = new Path(foo, "f1"); int numDeletedModified = 0; dfs.rename(foo, foo2); dfs.rename(bar, foo); dfs.rename(foo2, bar); DFSTestUtil.appendFile(dfs, foo_f1, (int) blockSize); numDeletedModified += 1; // modify ./bar/f1 return numDeletedModified; }
private int changeData7(Path dir) throws Exception { final Path foo = new Path(dir, "foo"); final Path foo2 = new Path(dir, "foo2"); final Path foo_f1 = new Path(foo, "f1"); final Path foo2_f2 = new Path(foo2, "f2"); final Path foo_d1 = new Path(foo, "d1"); final Path foo_d1_f3 = new Path(foo_d1, "f3"); int numCreatedModified = 0; dfs.rename(foo, foo2); DFSTestUtil.createFile(dfs, foo_f1, BLOCK_SIZE, DATA_NUM, 0L); numCreatedModified += 2; // create ./foo and ./foo/f1 DFSTestUtil.appendFile(dfs, foo_f1, (int) BLOCK_SIZE); dfs.rename(foo_f1, foo2_f2); numCreatedModified -= 1; // mv ./foo/f1 numCreatedModified += 2; // "M ./foo" and "+ ./foo/f2" DFSTestUtil.createFile(dfs, foo_d1_f3, BLOCK_SIZE, DATA_NUM, 0L); numCreatedModified += 2; // create ./foo/d1 and ./foo/d1/f3 return numCreatedModified; }
private int changeData7(Path dir) throws Exception { final Path foo = new Path(dir, "foo"); final Path foo2 = new Path(dir, "foo2"); final Path foo_f1 = new Path(foo, "f1"); final Path foo2_f2 = new Path(foo2, "f2"); final Path foo_d1 = new Path(foo, "d1"); final Path foo_d1_f3 = new Path(foo_d1, "f3"); int numDeletedAndModified = 0; dfs.rename(foo, foo2); DFSTestUtil.createFile(dfs, foo_f1, blockSize, dataNum, 0L); DFSTestUtil.appendFile(dfs, foo_f1, (int) blockSize); dfs.rename(foo_f1, foo2_f2); /* * Difference between snapshot s1 and current directory under directory /target: M . + ./foo R ./foo -> ./foo2 M ./foo + ./foo/f2 */ numDeletedAndModified += 1; // "M ./foo" DFSTestUtil.createFile(dfs, foo_d1_f3, blockSize, dataNum, 0L); return numDeletedAndModified; }
DFSTestUtil.createFile(dfs, f1, 2 * BLOCK_SIZE, DATA_NUM, 0); numCreatedModified += 1; // create ./foo/f1 DFSTestUtil.appendFile(dfs, f2, (int) BLOCK_SIZE); numCreatedModified += 1; // modify ./bar/f2 dfs.rename(bar, new Path(dir, "foo"));
numDeletedModified += 1; // delete ./foo/f1 DFSTestUtil.createFile(dfs, f1, 2 * blockSize, dataNum, 0); DFSTestUtil.appendFile(dfs, f2, (int) blockSize); numDeletedModified += 1; // modify ./bar/f2 dfs.rename(bar, new Path(dir, "foo"));
@Override public void prepare() throws Exception { // original size: 2.5 blocks DFSTestUtil.createFile(dfs, file, BLOCKSIZE * 2 + BLOCKSIZE / 2, REPLICATION, 0L); SnapshotTestHelper.createSnapshot(dfs, dir, "s1"); // truncate to 1.5 block dfs.truncate(file, BLOCKSIZE + BLOCKSIZE / 2); TestFileTruncate.checkBlockRecovery(file, dfs); // append another 1 BLOCK DFSTestUtil.appendFile(dfs, file, BLOCKSIZE); }
@Test(timeout=60000) public void testAddVolumesDuringWrite() throws IOException, InterruptedException, TimeoutException, ReconfigurationException { startDFSCluster(1, 1); String bpid = cluster.getNamesystem().getBlockPoolId(); Path testFile = new Path("/test"); createFile(testFile, 4); // Each volume has 2 blocks. addVolumes(2); // Continue to write the same file, thus the new volumes will have blocks. DFSTestUtil.appendFile(cluster.getFileSystem(), testFile, BLOCK_SIZE * 8); verifyFileLength(cluster.getFileSystem(), testFile, 8 + 4); // After appending data, there should be [2, 2, 4, 4] blocks in each volume // respectively. List<Integer> expectedNumBlocks = Arrays.asList(2, 2, 4, 4); List<Map<DatanodeStorage, BlockListAsLongs>> blockReports = cluster.getAllBlockReports(bpid); assertEquals(1, blockReports.size()); // 1 DataNode assertEquals(4, blockReports.get(0).size()); // 4 volumes Map<DatanodeStorage, BlockListAsLongs> dnReport = blockReports.get(0); List<Integer> actualNumBlocks = new ArrayList<Integer>(); for (BlockListAsLongs blockList : dnReport.values()) { actualNumBlocks.add(blockList.getNumberOfBlocks()); } Collections.sort(actualNumBlocks); assertEquals(expectedNumBlocks, actualNumBlocks); }
@Test(timeout=60000) public void testAddVolumesToFederationNN() throws IOException, TimeoutException, InterruptedException, ReconfigurationException { // Starts a Cluster with 2 NameNode and 3 DataNodes. Each DataNode has 2 // volumes. final int numNameNodes = 2; final int numDataNodes = 1; startDFSCluster(numNameNodes, numDataNodes); Path testFile = new Path("/test"); // Create a file on the first namespace with 4 blocks. createFile(0, testFile, 4); // Create a file on the second namespace with 4 blocks. createFile(1, testFile, 4); // Add 2 volumes to the first DataNode. final int numNewVolumes = 2; addVolumes(numNewVolumes); // Append to the file on the first namespace. DFSTestUtil.appendFile(cluster.getFileSystem(0), testFile, BLOCK_SIZE * 8); List<List<Integer>> actualNumBlocks = getNumBlocksReport(0); assertEquals(cluster.getDataNodes().size(), actualNumBlocks.size()); List<Integer> blocksOnFirstDN = actualNumBlocks.get(0); Collections.sort(blocksOnFirstDN); assertEquals(Arrays.asList(2, 2, 4, 4), blocksOnFirstDN); // Verify the second namespace also has the new volumes and they are empty. actualNumBlocks = getNumBlocksReport(1); assertEquals(4, actualNumBlocks.get(0).size()); assertEquals(numNewVolumes, Collections.frequency(actualNumBlocks.get(0), 0)); }
DFSTestUtil.appendFile(hdfs, file, BLOCKSIZE); DFSTestUtil.appendFile(hdfs, file, BLOCKSIZE); DFSTestUtil.appendFile(hdfs, file, BLOCKSIZE);
public void testGetFileChecksum(final Path foo, final int appendLength) throws Exception { final int appendRounds = 16; FileChecksum[] fc = new FileChecksum[appendRounds + 1]; DFSTestUtil.createFile(dfs, foo, appendLength, REPLICATION, 0L); fc[0] = dfs.getFileChecksum(foo); for (int i = 0; i < appendRounds; i++) { DFSTestUtil.appendFile(dfs, foo, appendLength); fc[i + 1] = dfs.getFileChecksum(foo); } for (int i = 0; i < appendRounds + 1; i++) { FileChecksum checksum = dfs.getFileChecksum(foo, appendLength * (i+1)); Assert.assertTrue(checksum.equals(fc[i])); } }
/** * Rename a file and then append some data to it */ @Test public void testDiffReportWithRenameAndAppend() throws Exception { final Path root = new Path("/"); final Path foo = new Path(root, "foo"); DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, seed); SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); final Path bar = new Path(root, "bar"); hdfs.rename(foo, bar); DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); // we always put modification on the file before rename verifyDiffReport(root, "s0", "s1", new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"), DFSUtil.string2Bytes("bar"))); }
assertTrue("File not created", fs.exists(fileName)); cluster.getDataNodes().get(1).shutdown(); DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock"); cluster.restartDataNode(1, true); GenericTestUtils.waitFor(new Supplier<Boolean>() { DFSTestUtil.appendFile(fs, fileName, "appendCorruptBlock"); runFsck(cluster.getConfiguration(0), 0, true, "/"); }finally {
DFSTestUtil.appendFile(getDFS(), bar, BLOCKSIZE);
DFSTestUtil.appendFile(dfs, createdFile1, appendLen); file1Len += appendLen;
.getSpaceConsumed().getStorageSpace(); try { DFSTestUtil.appendFile(getDFS(), file, BLOCKSIZE); Assert.fail("append didn't fail"); } catch (RemoteException e) {
.getSpaceConsumed().getStorageSpace(); try { DFSTestUtil.appendFile(getDFS(), file, BLOCKSIZE); Assert.fail("append didn't fail"); } catch (DSQuotaExceededException e) {