protected void testHdfsStreaming(Path filename) throws IOException { byte[] buf = new byte[1024]; FileSystem fs = filename.getFileSystem(getConf()); // read the file from start to finish Stopwatch fileOpenTimer = Stopwatch.createUnstarted(); Stopwatch streamTimer = Stopwatch.createUnstarted(); fileOpenTimer.start(); FSDataInputStream in = fs.open(filename); fileOpenTimer.stop(); long totalBytes = 0; streamTimer.start(); while (true) { int read = in.read(buf); if (read < 0) { break; } totalBytes += read; } streamTimer.stop(); double throughput = (double)totalBytes / streamTimer.elapsed(TimeUnit.SECONDS); System.out.println("HDFS streaming: "); System.out.println("total time to open: " + fileOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total time to read: " + streamTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")"); System.out.println("throghput : " + StringUtils.humanReadableInt((long)throughput) + "B/s"); }
scanTimer.stop();
scanTimer.stop();
protected void testHdfsStreaming(Path filename) throws IOException { byte[] buf = new byte[1024]; FileSystem fs = filename.getFileSystem(getConf()); // read the file from start to finish Stopwatch fileOpenTimer = Stopwatch.createUnstarted(); Stopwatch streamTimer = Stopwatch.createUnstarted(); fileOpenTimer.start(); FSDataInputStream in = fs.open(filename); fileOpenTimer.stop(); long totalBytes = 0; streamTimer.start(); while (true) { int read = in.read(buf); if (read < 0) { break; } totalBytes += read; } streamTimer.stop(); double throughput = (double)totalBytes / streamTimer.elapsed(TimeUnit.SECONDS); System.out.println("HDFS streaming: "); System.out.println("total time to open: " + fileOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total time to read: " + streamTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")"); System.out.println("throghput : " + StringUtils.humanReadableInt((long)throughput) + "B/s"); }
protected void testHdfsStreaming(Path filename) throws IOException { byte[] buf = new byte[1024]; FileSystem fs = filename.getFileSystem(getConf()); // read the file from start to finish Stopwatch fileOpenTimer = Stopwatch.createUnstarted(); Stopwatch streamTimer = Stopwatch.createUnstarted(); fileOpenTimer.start(); FSDataInputStream in = fs.open(filename); fileOpenTimer.stop(); long totalBytes = 0; streamTimer.start(); while (true) { int read = in.read(buf); if (read < 0) { break; } totalBytes += read; } streamTimer.stop(); double throughput = (double)totalBytes / streamTimer.elapsed(TimeUnit.SECONDS); System.out.println("HDFS streaming: "); System.out.println("total time to open: " + fileOpenTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total time to read: " + streamTimer.elapsed(TimeUnit.MILLISECONDS) + " ms"); System.out.println("total bytes: " + totalBytes + " bytes (" + StringUtils.humanReadableInt(totalBytes) + ")"); System.out.println("throghput : " + StringUtils.humanReadableInt((long)throughput) + "B/s"); }
scanTimer.stop();
scanTimer.stop();
scanTimer.stop();
scanTimer.stop();
stopwatch.stop(); if (!(service instanceof NoOpService)) { logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});