/** * Load an edit log, and apply the changes to the in-memory structure * This is where we apply edits that we've been writing to disk all * along. */ int loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { long startTime = now(); currentTxId = expectedStartingTxId; int numEdits = loadFSEdits(edits, true); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (now()-startTime)/1000 + " seconds."); return numEdits; }
private long dispatchBlockMoves() throws InterruptedException { long bytesLastMoved = bytesMoved.get(); Future<?>[] futures = new Future<?>[sources.size()]; int i=0; for (Source source : sources) { futures[i++] = dispatcherExecutor.submit( source.new BlockMoveDispatcher(Util.now())); } // wait for all dispatcher threads to finish for (Future<?> future : futures) { try { future.get(); } catch (ExecutionException e) { LOG.warn("Dispatcher thread failed", e.getCause()); } } // wait for all block moving to be done waitForMoveCompletion(); return bytesMoved.get()-bytesLastMoved; }
public int run(String[] args) throws Exception { final long startTime = Util.now(); try { checkReplicationPolicyCompatibility(conf); final List<InetSocketAddress> namenodes = DFSUtil.getClientRpcAddresses(conf, null); parse(args); return Balancer.run(namenodes, conf); } catch (IOException e) { System.out.println(e + ". Exiting ..."); return IO_EXCEPTION; } catch (InterruptedException e) { System.out.println(e + ". Exiting ..."); return INTERRUPTED; } catch (Exception e) { e.printStackTrace(); return ILLEGAL_ARGS; } finally { System.out.println("Balancing took " + time2Str(Util.now()-startTime)); } }
public void testThrottler() throws IOException { Configuration conf = new HdfsConfiguration(); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); long bandwidthPerSec = 1024*1024L; final long TOTAL_BYTES =6*bandwidthPerSec; long bytesToSend = TOTAL_BYTES; long start = Util.now(); DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec); long totalBytes = 0L; long bytesSent = 1024*512L; // 0.5MB throttler.throttle(bytesSent); bytesToSend -= bytesSent; bytesSent = 1024*768L; // 0.75MB throttler.throttle(bytesSent); bytesToSend -= bytesSent; try { Thread.sleep(1000); } catch (InterruptedException ignored) {} throttler.throttle(bytesToSend); long end = Util.now(); assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec); }
if (Util.now()-startTime > maxIterationTime) { isTimeUp = true; continue;