private void checkFile(FileSystem fileSys, Path name, int repl) throws IOException { DFSTestUtil.waitReplication(fileSys, name, (short) repl); }
private void checkFile(FileSystem fileSys, Path name, int repl) throws IOException { DFSTestUtil.waitReplication(fileSys, name, (short) repl); }
private static void writeDataAndSetReplication(FileSystem fileSys, Path name, OutputStream out, short replication, int numBlocks) throws IOException { for (int i = 0; i < numBlocks; i++) { out.write(databuf); } out.close(); DFSTestUtil.waitReplication(fileSys, name, replication); }
static void writeFile(Configuration conf, Path name, short replication, int numBlocks) throws IOException { FileSystem fileSys = FileSystem.get(conf); FSDataOutputStream stm = fileSys.create(name, true, conf.getInt("io.file.buffer.size", 4096), replication, (long)BLOCKSIZE); for (int i = 0; i < numBlocks; i++) { stm.write(databuf); } stm.close(); DFSTestUtil.waitReplication(fileSys, name, replication); }
public void waitReplication(FileSystem fs, String topdir, short value) throws IOException { Path root = new Path(topdir); /** wait for the replication factor to settle down */ for (int idx = 0; idx < nFiles; idx++) { waitReplication(fs, new Path(root, files[idx].getName()), value); } }
private void checkFile(FileSystem fileSys, Path name, int repl) throws IOException, InterruptedException, TimeoutException { DFSTestUtil.waitReplication(fileSys, name, (short) repl); }
private void createFile(long fileLen, short replicationFactor) throws IOException { FileSystem fs = cluster.getFileSystem(); DFSTestUtil.createFile(fs, filePath, fileLen, replicationFactor, r.nextLong()); DFSTestUtil.waitReplication(fs, filePath, replicationFactor); }
private static void writeDataAndSetReplication(FileSystem fileSys, Path name, OutputStream out, short replication, int numBlocks) throws IOException, TimeoutException, InterruptedException { for (int i = 0; i < numBlocks; i++) { out.write(databuf); } out.close(); DFSTestUtil.waitReplication(fileSys, name, replication); }
static void writeFile(NameNode namenode, Configuration conf, Path name, short replication) throws IOException { FileSystem fileSys = FileSystem.get(conf); SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, name, BytesWritable.class, BytesWritable.class, CompressionType.NONE); writer.append(new BytesWritable(), new BytesWritable()); writer.close(); fileSys.setReplication(name, replication); DFSTestUtil.waitReplication(fileSys, name, replication); }
private void startDNandWait(Path filePath, boolean waitReplicas) throws IOException { if(LOG.isDebugEnabled()) { LOG.debug("Before next DN start: " + cluster.getDataNodes().size()); } cluster.startDataNodes(conf, 1, true, null, null); ArrayList<DataNode> datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 2); if(LOG.isDebugEnabled()) { LOG.debug("New datanode " + cluster.getDataNodes().get(datanodes.size() - 1) .getDatanodeRegistration() + " has been started"); } if (waitReplicas) DFSTestUtil.waitReplication(fs, filePath, REPL_FACTOR); }
static void writeFile(Configuration conf, Path name, short replication, int numBlocks) throws IOException, TimeoutException, InterruptedException { FileSystem fileSys = FileSystem.get(conf); FSDataOutputStream stm = fileSys.create(name, true, conf.getInt("io.file.buffer.size", 4096), replication, (long)BLOCKSIZE); for (int i = 0; i < numBlocks; i++) { stm.write(databuf); } stm.close(); DFSTestUtil.waitReplication(fileSys, name, replication); }
public void waitReplication(FileSystem fs, String topdir, short value) throws IOException, InterruptedException, TimeoutException { Path root = new Path(topdir); /** wait for the replication factor to settle down */ for (int idx = 0; idx < nFiles; idx++) { waitReplication(fs, new Path(root, files[idx].getName()), value); } }
private void createInputs(FileSystem fs, Path inDir, String fileName) throws IOException, TimeoutException, InterruptedException { // create a multi-block file on hdfs Path path = new Path(inDir, fileName); final short replication = 2; DataOutputStream out = fs.create(path, true, 4096, replication, 512, null); for(int i=0; i < 1000; ++i) { out.writeChars("Hello\n"); } out.close(); System.out.println("Wrote file"); DFSTestUtil.waitReplication(fs, path, replication); }
private static void createFile(Suite s, int index, long len ) throws IOException, InterruptedException, TimeoutException { final FileSystem fs = s.cluster.getFileSystem(index); DFSTestUtil.createFile(fs, FILE_PATH, len, s.replication, RANDOM.nextLong()); DFSTestUtil.waitReplication(fs, FILE_PATH, s.replication); }
static void createFile(MiniDFSCluster cluster, Path filePath, long fileLen, short replicationFactor, int nnIndex) throws IOException, InterruptedException, TimeoutException { FileSystem fs = cluster.getFileSystem(nnIndex); DFSTestUtil.createFile(fs, filePath, fileLen, replicationFactor, r.nextLong()); DFSTestUtil.waitReplication(fs, filePath, replicationFactor); }
/** create a file with a length of <code>fileLen</code> */ private void createFile(int index, long len ) throws IOException, InterruptedException, TimeoutException { final FileSystem fs = cluster.getFileSystem(index); DFSTestUtil.createFile(fs, FILE_PATH, len, replication, RANDOM.nextLong()); DFSTestUtil.waitReplication(fs, FILE_PATH, replication); }
private void createFile(int fsIdx, Path path, int numBlocks, short replicateFactor) throws IOException, TimeoutException, InterruptedException { final int seed = 0; final DistributedFileSystem fs = cluster.getFileSystem(fsIdx); DFSTestUtil.createFile(fs, path, BLOCK_SIZE * numBlocks, replicateFactor, seed); DFSTestUtil.waitReplication(fs, path, replicateFactor); }
static void writeFile(NameNode namenode, Configuration conf, Path name, short replication) throws IOException, TimeoutException, InterruptedException { FileSystem fileSys = FileSystem.get(conf); SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, name, BytesWritable.class, BytesWritable.class, CompressionType.NONE); writer.append(new BytesWritable(), new BytesWritable()); writer.close(); fileSys.setReplication(name, replication); DFSTestUtil.waitReplication(fileSys, name, replication); }
private BlockLocation[] getBlockLocations(Path p) throws Exception { DFSTestUtil.waitReplication(dfs, p, (short)3); BlockLocation[] locations = dfs.getClient().getBlockLocations( p.toUri().getPath(), 0, Long.MAX_VALUE); assertTrue(locations.length == 1 && locations[0].getHosts().length == 3); return locations; }
private void startDNandWait(Path filePath, boolean waitReplicas) throws IOException, InterruptedException, TimeoutException { if (LOG.isDebugEnabled()) { LOG.debug("Before next DN start: " + cluster.getDataNodes().size()); } cluster.startDataNodes(conf, 1, true, null, null); cluster.waitClusterUp(); ArrayList<DataNode> datanodes = cluster.getDataNodes(); assertEquals(datanodes.size(), 2); if (LOG.isDebugEnabled()) { int lastDn = datanodes.size() - 1; LOG.debug("New datanode " + cluster.getDataNodes().get(lastDn).getDisplayName() + " has been started"); } if (waitReplicas) { DFSTestUtil.waitReplication(fs, filePath, REPL_FACTOR); } }