.manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.REGULAR) .build(); fail("Was able to start NN from 0.3.0 image");
public void testUpgradeFromImage() throws IOException { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Ant System.setProperty("test.build.data", "build/test/data"); } conf.setInt("dfs.datanode.scan.period.hours", -1); // block scanning off cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes) .format(false) .startupOption(StartupOption.UPGRADE) .build(); cluster.waitActive(); DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); DFSClient dfsClient = dfs.dfs; //Safemode will be off only after upgrade is complete. Wait for it. while ( dfsClient.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET) ) { LOG.info("Waiting for SafeMode to be OFF."); try { Thread.sleep(1000); } catch (InterruptedException ignored) {} } verifyFileSystem(dfs); } finally { if (cluster != null) { cluster.shutdown(); } } }
/** * Attempts to start a NameNode with the given operation. Starting * the NameNode should throw an exception. */ void startNameNodeShouldFail(StartupOption operation, String searchString) { try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) .startupOption(operation) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .build(); // should fail throw new AssertionError("NameNode should have failed to start"); } catch (Exception expected) { if (!expected.getMessage().contains(searchString)) { fail("Expected substring '" + searchString + "' in exception " + "but got: " + StringUtils.stringifyException(expected)); } // expected } }
.manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.REGULAR) .build(); fail("Was able to start NN from 0.3.0 image");
.numDataNodes(0) .waitSafeMode(false) .startupOption(StartupOption.UPGRADE) .build(); try {
.manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.REGULAR) .build(); cluster.finalizeCluster(conf);
try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) .startupOption(operation) .format(false) .manageDataDfsDirs(false)
/** * Starts DFS as specified in member-variable options. Also writes out * configuration and details, if requested. */ public void start() throws IOException, FileNotFoundException { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nameNodePort) .numDataNodes(numDataNodes) .startupOption(dfsOpts) .format(format) .build(); dfs.waitActive(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new File(writeConfig)); conf.writeXml(fos); fos.close(); } if (writeDetails != null) { Map<String, Object> map = new TreeMap<String, Object>(); if (dfs != null) { map.put("namenode_port", dfs.getNameNodePort()); } FileWriter fw = new FileWriter(new File(writeDetails)); fw.write(new JSON().toJSON(map)); fw.close(); } }
/** * start with -importCheckpoint option and verify that the files are in separate directories and of the right length * @throws IOException */ private void checkNameNodeFiles() throws IOException{ // start namenode with import option LOG.info("-- about to start DFS cluster"); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(config) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(IMPORT).build(); cluster.waitActive(); LOG.info("--NN started with checkpoint option"); NameNode nn = cluster.getNameNode(); assertNotNull(nn); // Verify that image file sizes did not change. FSImage image = nn.getFSImage(); verifyDifferentDirs(image, this.fsimageLength, this.editsLength); } finally { if(cluster != null) cluster.shutdown(); } }
/** * start with -importCheckpoint option and verify that the files are in separate directories and of the right length * @throws IOException */ private void checkNameNodeFiles() throws IOException{ // start namenode with import option LOG.info("-- about to start DFS cluster"); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(config) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(IMPORT).build(); cluster.waitActive(); LOG.info("--NN started with checkpoint option"); NameNode nn = cluster.getNameNode(); assertNotNull(nn); // Verify that image file sizes did not change. FSImage image = nn.getFSImage(); verifyDifferentDirs(image, this.fsimageLength, this.editsLength); } finally { if(cluster != null) cluster.shutdown(); } }
void upgradeAndVerify(MiniDFSCluster.Builder bld, ClusterVerifier verifier) throws IOException { MiniDFSCluster cluster = null; try { bld.format(false).startupOption(StartupOption.UPGRADE) .clusterId("testClusterId"); cluster = bld.build(); cluster.waitActive(); DistributedFileSystem dfs = cluster.getFileSystem(); DFSClient dfsClient = dfs.dfs; //Safemode will be off only after upgrade is complete. Wait for it. while ( dfsClient.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET) ) { LOG.info("Waiting for SafeMode to be OFF."); try { Thread.sleep(1000); } catch (InterruptedException ignored) {} } recoverAllLeases(dfsClient, new Path("/")); verifyFileSystem(dfs); if (verifier != null) { verifier.verifyClusterPostUpgrade(cluster); } } finally { if (cluster != null) { cluster.shutdown(); } } }
/** * Attempts to start a NameNode with the given operation. Starting * the NameNode should throw an exception. */ void startNameNodeShouldFail(StartupOption operation) { try { //cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).startupOption(operation).build(); // should fail // we set manage dirs to true as NN has to start from untar'ed image with // nn dirs set to name1 and name2 cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) .format(false) .startupOption(operation) .build(); // should fail throw new AssertionError("Jakob was here. NameNode should have failed to start"); } catch (Exception expected) { expected = null; // expected } }
private FileChecksum writeUnencryptedAndThenRestartEncryptedCluster() throws IOException { cluster = new MiniDFSCluster.Builder(conf).build(); fs = getFileSystem(conf); writeTestDataToFile(fs); assertEquals(PLAIN_TEXT, DFSTestUtil.readFile(fs, TEST_PATH)); FileChecksum checksum = fs.getFileChecksum(TEST_PATH); fs.close(); cluster.shutdown(); setEncryptionConfigKeys(); cluster = new MiniDFSCluster.Builder(conf) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .format(false) .startupOption(StartupOption.REGULAR) .build(); fs = getFileSystem(conf); return checksum; }
/** * Attempts to start a NameNode with the given operation. Starting * the NameNode should throw an exception. */ void startNameNodeShouldFail(StartupOption operation) { try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) .startupOption(operation) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .build(); // should fail throw new AssertionError("NameNode should have failed to start"); } catch (Exception expected) { // expected } }
static void restartCluster(StartupOption o) throws IOException { cluster.shutdown(); if(StartupOption.ROLLBACK == o) NameNode.doRollback(conf, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM) .format(false) .nameNodePort(NameNode.DEFAULT_PORT) .startupOption(o==StartupOption.ROLLBACK ? StartupOption.REGULAR : o) .dnStartupOption(o!=StartupOption.ROLLBACK ? StartupOption.REGULAR : o) .build(); fs = cluster.getFileSystem(); } }
public void setup(String basePath) throws Exception { conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/core-site.xml")); conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml")); conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); cleanupLocal(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR); MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf); build.nameNodePort(nameNodePort); build.numDataNodes(numDataNodes); build.startupOption(StartupOption.REGULAR); dfsCluster = build.build(); dfs = FileSystem.get(conf); loadData(basePath); }
/** * Create an instance of a newly configured cluster for testing that does * not manage its own directories or files */ private MiniDFSCluster createCluster() throws IOException { return new MiniDFSCluster.Builder(conf).numDataNodes(0) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.UPGRADE) .build(); }
private MiniDFSCluster createCluster(Configuration c) throws IOException { return new MiniDFSCluster.Builder(c) .numDataNodes(0) .startupOption(StartupOption.REGULAR) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .build(); } /**
private MiniDFSCluster createCluster(Configuration c) throws IOException { return new MiniDFSCluster.Builder(c) .numDataNodes(0) .startupOption(StartupOption.REGULAR) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .build(); } /**
/** * Create an instance of a newly configured cluster for testing that does * not manage its own directories or files */ private MiniDFSCluster createCluster() throws IOException { return new MiniDFSCluster.Builder(conf).numDataNodes(0) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.UPGRADE) .build(); }