.format(false) .racks(racks) .simulatedCapacities(capacities) .build(); cluster.waitActive();
.numDataNodes(nDataNodes) .racks(racks) .simulatedCapacities(capacities) .build(); LOG.info("RUN_TEST 0");
.numDataNodes(nDataNodes) .racks(racks) .simulatedCapacities(capacities) .build(); LOG.info("UNEVEN 2"); .numDataNodes(nDataNodes) .racks(racks) .simulatedCapacities(capacities) .format(false) .build();
int numOfDatanodes = capacities.length; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length) .racks(racks).simulatedCapacities(capacities).build(); try { cluster.waitActive();
.numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); HATestUtil.setFailoverConfigurations(cluster, conf);
.hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
.format(false) .racks(racks) .simulatedCapacities(capacities) .build(); try {
.numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder);
.numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder);
.numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities); MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups); cluster = new MiniDFSClusterWithNodeGroup(builder);
private void testBalancerDefaultConstructor(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack) throws Exception { int numOfDatanodes = capacities.length; assertEquals(numOfDatanodes, racks.length); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); try { cluster.waitActive(); client = DFSClient.createNamenode(conf); long totalCapacity = 0L; for (long capacity : capacities) { totalCapacity += capacity; } // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; createFile(totalUsedSpace / numOfDatanodes, (short) numOfDatanodes); // start up an empty node with the same capacity and on the same rack cluster.startDataNodes(conf, 1, true, null, new String[] { newRack }, new long[] { newCapacity }); totalCapacity += newCapacity; // run balancer and validate results runBalancerDefaultConstructor(conf, totalUsedSpace, totalCapacity); } finally { cluster.shutdown(); } }
private void testBalancerDefaultConstructor(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack) throws Exception { int numOfDatanodes = capacities.length; assertEquals(numOfDatanodes, racks.length); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); try { cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); long totalCapacity = sum(capacities); // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity * 3 / 10; createFile(cluster, filePath, totalUsedSpace / numOfDatanodes, (short) numOfDatanodes, 0); // start up an empty node with the same capacity and on the same rack cluster.startDataNodes(conf, 1, true, null, new String[] { newRack }, new long[] { newCapacity }); totalCapacity += newCapacity; // run balancer and validate results runBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.shutdown(); } }
private void test(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack) throws Exception { int numOfDatanodes = capacities.length; assertEquals(numOfDatanodes, racks.length); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(capacities.length) .racks(racks) .simulatedCapacities(capacities) .build(); try { cluster.waitActive(); client = DFSClient.createNamenode(conf); long totalCapacity=0L; for(long capacity:capacities) { totalCapacity += capacity; } // fill up the cluster to be 30% full long totalUsedSpace = totalCapacity*3/10; createFile(totalUsedSpace/numOfDatanodes, (short)numOfDatanodes); // start up an empty node with the same capacity and on the same rack cluster.startDataNodes(conf, 1, true, null, new String[]{newRack}, new long[]{newCapacity}); totalCapacity += newCapacity; // run balancer and validate results runBalancer(conf, totalUsedSpace, totalCapacity); } finally { cluster.shutdown(); } }
private void testUnevenDistribution(Configuration conf, long distribution[], long capacities[], String[] racks) throws Exception { int numDatanodes = distribution.length; if (capacities.length != numDatanodes || racks.length != numDatanodes) { throw new IllegalArgumentException("Array length is not the same"); } // calculate total space that need to be filled final long totalUsedSpace = sum(distribution); // fill the cluster ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace, (short) numDatanodes); // redistribute blocks Block[][] blocksDN = distributeBlocks( blocks, (short)(numDatanodes-1), distribution); // restart the cluster: do NOT format the cluster conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) .format(false) .racks(racks) .simulatedCapacities(capacities) .build(); cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); for(int i = 0; i < blocksDN.length; i++) cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null); final long totalCapacity = sum(capacities); runBalancer(conf, totalUsedSpace, totalCapacity); cluster.shutdown(); }