miniDFSCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes).format(format) .racks(racks).nnTopology(topo).build(); miniDFSCluster.waitActive(); miniDFSCluster.transitionToActive(0); miniDFSCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDataNodes).format(format) .racks(racks).build();
public static MiniDFSCluster getLocalHDFSCluster() throws Exception { setHadoopHomeWindows(); Configuration conf = new HdfsConfiguration(); conf.set("fs.defaultFS", "hdfs://localhost"); File hdfsPath = new File(System.getProperty("user.dir") + File.separator + "hadoop" + File.separator + "hdfs"); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsPath.getAbsolutePath()); MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(conf) .nameNodePort(12345) .nameNodeHttpPort(12341) .numDataNodes(1) .storagesPerDatanode(2) .format(true) .racks(null) .build(); miniDFSCluster.waitActive(); return miniDFSCluster; } }
@Override public void start() throws Exception { LOG.info("HDFS: Starting MiniDfsCluster"); configure(); miniDFSCluster = new MiniDFSCluster.Builder(hdfsConfig) .nameNodePort(hdfsNamenodePort) .nameNodeHttpPort(hdfsNamenodeHttpPort==null? 0 : hdfsNamenodeHttpPort.intValue() ) .numDataNodes(hdfsNumDatanodes) .format(hdfsFormat) .racks(null) .build(); }
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) .format(false) .racks(racks) .simulatedCapacities(capacities) .build();
private void test(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack) throws Exception { int numOfDatanodes = capacities.length; assertEquals(numOfDatanodes, racks.length); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); try { cluster.waitActive(); client = DFSClient.createNamenode(conf); long totalCapacity=0L; for(long capacity:capacities) { totalCapacity += capacity; } // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity*3/10; createFile(totalUsedSpace/numOfDatanodes, (short)numOfDatanodes); // start up an empty node with the same capacity and on the same rack cluster.startDataNodes(conf, 1, true, null, new String[]{newRack}, new long[]{newCapacity}); totalCapacity += newCapacity; // run balancer and validate results runBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.shutdown(); } }
@Test public void testUnderReplicatedUsesNewRacks() throws Exception { Configuration conf = getConf(); short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); // All datanodes are on the same rack String racks[] = {"/rack1", "/rack1", "/rack1", "/rack1", "/rack1"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); try { // Create a file with one block final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); Block b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1); // Add new datanodes on a different rack and increase the // replication factor so the block is underreplicated and make // sure at least one of the hosts on the new rack is used. String newRacks[] = {"/rack2", "/rack2"}; cluster.startDataNodes(conf, 2, true, null, newRacks); REPLICATION_FACTOR = 5; ns.setReplication("/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
/** * Starts the cluster within a testcase. * <p/> * Note that the cluster is already started when the testcase method * is invoked. This method is useful if as part of the testcase the * cluster has to be shutdown and restarted again. * <p/> * If the cluster is already running this method does nothing. * * @param reformatDFS indicates if DFS has to be reformated * @param props configuration properties to inject to the mini cluster * @throws Exception if the cluster could not be started */ protected synchronized void startCluster(boolean reformatDFS, Properties props) throws Exception { if (dfsCluster == null) { JobConf conf = new JobConf(); if (props != null) { for (Map.Entry entry : props.entrySet()) { conf.set((String) entry.getKey(), (String) entry.getValue()); } } dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) .format(reformatDFS).racks(null).build(); ConfigurableMiniMRCluster.setConfiguration(props); //noinspection deprecation mrCluster = new ConfigurableMiniMRCluster(2, getFileSystem().getUri().toString(), 1, conf); } }
private void testUnevenDistribution(Configuration conf, long distribution[], long capacities[], String[] racks) throws Exception { int numDatanodes = distribution.length; if (capacities.length != numDatanodes || racks.length != numDatanodes) { throw new IllegalArgumentException("Array length is not the same"); } // calculate total space that need to be filled final long totalUsedSpace = sum(distribution); // fill the cluster ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace, (short) numDatanodes); // redistribute blocks Block[][] blocksDN = distributeBlocks( blocks, (short)(numDatanodes-1), distribution); // restart the cluster: do NOT format the cluster conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) .format(false) .racks(racks) .simulatedCapacities(capacities) .build(); cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); for(int i = 0; i < blocksDN.length; i++) cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null); final long totalCapacity = sum(capacities); runBalancer(conf, totalUsedSpace, totalCapacity); cluster.shutdown(); }
private void testBalancerDefaultConstructor(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack) throws Exception { int numOfDatanodes = capacities.length; assertEquals(numOfDatanodes, racks.length); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); try { cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); long totalCapacity = sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; createFile(cluster, filePath, totalUsedSpace / numOfDatanodes, (short) numOfDatanodes, 0); // start up an empty node with the same capacity and on the same rack cluster.startDataNodes(conf, 1, true, null, new String[] { newRack }, new long[] { newCapacity }); totalCapacity += newCapacity; // run balancer and validate results runBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.shutdown(); } }
@BeforeClass public static void setup() throws IOException { try { dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) .format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(TestMRJobs.class.getName(), NUM_NODE_MGRS); Configuration conf = new Configuration(); conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); mrCluster.init(conf); mrCluster.start(); } // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to // workaround the absent public discache. localFs.copyFromLocalFile(new Path(MiniMRYarnCluster.APPJAR), APP_JAR); localFs.setPermission(APP_JAR, new FsPermission("700")); }
@Test public void testReduceReplFactorRespectsRackPolicy() throws Exception { Configuration conf = getConf(); short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); String racks[] = {"/rack1", "/rack1", "/rack2", "/rack2"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); try { // Create a file with one block final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); Block b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); // Decrease the replication factor, make sure the deleted replica // was not the one that lived on the rack with only one replica, // ie we should still have 2 racks after reducing the repl factor. REPLICATION_FACTOR = 2; ns.setReplication("/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Test public void testReduceReplFactorRespectsRackPolicy() throws Exception { Configuration conf = getConf(); short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); String racks[] = {"/rack1", "/rack1", "/rack2", "/rack2"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); try { // Create a file with one block final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); // Decrease the replication factor, make sure the deleted replica // was not the one that lived on the rack with only one replica, // ie we should still have 2 racks after reducing the repl factor. REPLICATION_FACTOR = 2; NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Test public void testSufficientlyReplBlocksUsesNewRack() throws Exception { Configuration conf = getConf(); final short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); // All datanodes are on the same rack String racks[] = {"/rack1", "/rack1", "/rack1"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); try { // Create a file with one block with a replication factor of 3 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0); // Add a new datanode on a different rack String newRacks[] = {"/rack2"}; cluster.startDataNodes(conf, 1, true, null, newRacks); cluster.waitActive(); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Test public void testSufficientlyReplBlocksUsesNewRack() throws Exception { Configuration conf = getConf(); final short REPLICATION_FACTOR = 3; final Path filePath = new Path("/testFile"); // All datanodes are on the same rack String racks[] = {"/rack1", "/rack1", "/rack1"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); try { // Create a file with one block with a replication factor of 3 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); Block b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 1); // Add a new datanode on a different rack String newRacks[] = {"/rack2"}; cluster.startDataNodes(conf, 1, true, null, newRacks); cluster.waitActive(); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Before @Override public void setUp() throws Exception { super.setUp(); conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG, HDFSPolicyProvider.class, PolicyProvider.class); // Many of the tests expect a replication value of 1 in the output conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); // Build racks and hosts configuration to test dfsAdmin -printTopology String [] racks = {"/rack1", "/rack1", "/rack2", "/rack2", "/rack2", "/rack3", "/rack4", "/rack4" }; String [] hosts = {"host1", "host2", "host3", "host4", "host5", "host6", "host7", "host8" }; dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(8) .racks(racks) .hosts(hosts) .build(); dfsCluster.waitClusterUp(); namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///"); username = System.getProperty("user.name"); fs = dfsCluster.getFileSystem(); assertTrue("Not a HDFS: "+fs.getUri(), fs instanceof DistributedFileSystem); }
@Test public void testSufficientlySingleReplBlockUsesNewRack() throws Exception { Configuration conf = getConf(); short REPLICATION_FACTOR = 1; final Path filePath = new Path("/testFile"); String racks[] = {"/rack1", "/rack1", "/rack1", "/rack2"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); try { // Create a file with one block with a replication factor of 1 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); Block b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0); REPLICATION_FACTOR = 2; ns.setReplication("/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@Test public void testSufficientlySingleReplBlockUsesNewRack() throws Exception { Configuration conf = getConf(); short REPLICATION_FACTOR = 1; final Path filePath = new Path("/testFile"); String racks[] = {"/rack1", "/rack1", "/rack1", "/rack2"}; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(racks.length).racks(racks).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); try { // Create a file with one block with a replication factor of 1 final FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, 1L, REPLICATION_FACTOR, 1L); ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, filePath); DFSTestUtil.waitForReplication(cluster, b, 1, REPLICATION_FACTOR, 0); REPLICATION_FACTOR = 2; NameNodeAdapter.setReplication(ns, "/testFile", REPLICATION_FACTOR); DFSTestUtil.waitForReplication(cluster, b, 2, REPLICATION_FACTOR, 0); } finally { cluster.shutdown(); } }
@BeforeClass public static void setUp() throws Exception { final Configuration conf = new Configuration(); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); conf.set(YarnConfiguration.RM_PRINCIPAL, "jt_id/" + SecurityUtil.HOSTNAME_PATTERN + "@APACHE.ORG"); final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); builder.checkExitOnShutdown(true); builder.numDataNodes(numSlaves); builder.format(true); builder.racks(null); dfsCluster = builder.build(); mrCluster = new MiniMRYarnCluster(TestBinaryTokenFile.class.getName(), noOfNMs); mrCluster.init(conf); mrCluster.start(); NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); FileSystem fs = dfsCluster.getFileSystem(); p1 = new Path("file1"); p1 = fs.makeQualified(p1); }
@Before public void setup() throws IOException { StaticMapping.resetMap(); Configuration conf = new HdfsConfiguration(); final String[] racks = { "/RACK0", "/RACK0", "/RACK2", "/RACK3", "/RACK2" }; final String[] hosts = { "/host0", "/host1", "/host2", "/host3", "/host4" }; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE / 2); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(5).racks(racks) .hosts(hosts).build(); cluster.waitActive(); nameNodeRpc = cluster.getNameNodeRpc(); namesystem = cluster.getNamesystem(); perm = new PermissionStatus("TestDefaultBlockPlacementPolicy", null, FsPermission.getDefault()); }
private MiniDFSCluster newDFSCluster(JobConf conf) throws Exception { return new MiniDFSCluster.Builder(conf).numDataNodes(4) .racks(new String[]{"/rack0", "/rack0", "/rack1", "/rack1"}) .hosts(new String[]{"host0", "host1", "host2", "host3"}) .build(); }