double curMBPerSec = rateLimiter.getMBPerSec(); mergeThread.getName(), bytesToMB(merge.estimatedMergeBytes), bytesToMB(rateLimiter.getTotalBytesWritten()), nsToSec(now - mergeStartNS), nsToSec(rateLimiter.getTotalStoppedNS()), nsToSec(rateLimiter.getTotalPausedNS()), rateToString(rateLimiter.getMBPerSec()))); rateLimiter.setMBPerSec(newMBPerSec);
/** Sole constructor. */ public MergeThread(IndexWriter writer, OneMerge merge) { this.writer = writer; this.merge = merge; this.rateLimiter = new MergeRateLimiter(merge.getMergeProgress()); }
@Override public long pause(long bytes) throws MergePolicy.MergeAbortedException { totalBytesWritten.addAndGet(bytes); // While loop because we may wake up and check again when our rate limit // is changed while we were pausing: long paused = 0; long delta; while ((delta = maybePause(bytes, System.nanoTime())) >= 0) { // Keep waiting. paused += delta; } return paused; }
merge.rateLimiter.checkAbort(); context); merge.rateLimiter.checkAbort(); double sec = (t1-merge.mergeStartNS)/1000000000.; double segmentMB = (merge.info.sizeInBytes()/1024./1024.); double stoppedSec = merge.rateLimiter.getTotalStoppedNS()/1000000000.; double throttleSec = merge.rateLimiter.getTotalPausedNS()/1000000000.; infoStream.message("IW", "merge codec=" + codec + " maxDoc=" + merge.info.info.maxDoc() + "; merged segment has " + (mergeState.mergeFieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " + } catch (Throwable t) { synchronized(this) { if (merge.rateLimiter.getAbort()) { if (merge.rateLimiter.getAbort()) { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "abort merge after building CFS");
/** * The current MB per second rate limit for this merge. **/ public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) { if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getMBPerSec(); } assert false: "this is not merge thread"; return Double.POSITIVE_INFINITY; }
/** * Returns total bytes written by this merge. **/ public static long getTotalBytesWritten(Thread thread, MergePolicy.OneMerge merge) throws IOException { /** * TODO: The number of bytes written during the merge should be accessible in OneMerge. */ if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter .getTotalBytesWritten(); } assert false: "this is not merge thread"; return merge.totalBytesSize(); }
/** Sole constructor. */ public MergeRateLimiter(OneMergeProgress mergeProgress) { // Initially no IO limit; use setter here so minPauseCheckBytes is set: this.mergeProgress = mergeProgress; setMBPerSec(Double.POSITIVE_INFINITY); }
merge.rateLimiter.checkAbort(); context); merge.rateLimiter.checkAbort(); double sec = (t1-merge.mergeStartNS)/1000000000.; double segmentMB = (merge.info.sizeInBytes()/1024./1024.); double stoppedSec = merge.rateLimiter.getTotalStoppedNS()/1000000000.; double throttleSec = merge.rateLimiter.getTotalPausedNS()/1000000000.; infoStream.message("IW", "merge codec=" + codec + " maxDoc=" + merge.info.info.maxDoc() + "; merged segment has " + (mergeState.mergeFieldInfos.hasVectors() ? "vectors" : "no vectors") + "; " + } catch (Throwable t) { synchronized(this) { if (merge.rateLimiter.getAbort()) { if (merge.rateLimiter.getAbort()) { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "abort merge after building CFS");
/** * The current MB per second rate limit for this merge. **/ public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) { if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getMBPerSec(); } assert false: "this is not merge thread"; return Double.POSITIVE_INFINITY; }
/** * Returns total bytes written by this merge. **/ public static long getTotalBytesWritten(Thread thread, MergePolicy.OneMerge merge) throws IOException { /** * TODO: The number of bytes written during the merge should be accessible in OneMerge. */ if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter .getTotalBytesWritten(); } assert false: "this is not merge thread"; return merge.totalBytesSize(); }
rate = targetMBPerSec; rateLimiter.setMBPerSec(rate); targetMBPerSecChanged();
double curMBPerSec = rateLimiter.getMBPerSec(); mergeThread.getName(), bytesToMB(merge.estimatedMergeBytes), bytesToMB(rateLimiter.getTotalBytesWritten()), nsToSec(now - mergeStartNS), nsToSec(rateLimiter.getTotalStoppedNS()), nsToSec(rateLimiter.getTotalPausedNS()), rateToString(rateLimiter.getMBPerSec()))); rateLimiter.setMBPerSec(newMBPerSec);
/** * The current MB per second rate limit for this merge. **/ public static double getMbPerSec(Thread thread, MergePolicy.OneMerge merge) { if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter.getMBPerSec(); } assert false: "this is not merge thread"; return Double.POSITIVE_INFINITY; }
/** * Returns total bytes written by this merge. **/ public static long getTotalBytesWritten(Thread thread, MergePolicy.OneMerge merge) throws IOException { /** * TODO: The number of bytes written during the merge should be accessible in OneMerge. */ if (thread instanceof ConcurrentMergeScheduler.MergeThread) { return ((ConcurrentMergeScheduler.MergeThread) thread).rateLimiter .getTotalBytesWritten(); } assert false: "this is not merge thread"; return merge.totalBytesSize(); }
/** Sole constructor. */ public MergeRateLimiter(OneMergeProgress mergeProgress) { // Initially no IO limit; use setter here so minPauseCheckBytes is set: this.mergeProgress = mergeProgress; setMBPerSec(Double.POSITIVE_INFINITY); }
/** Sole constructor. */ public MergeThread(IndexWriter writer, OneMerge merge) { this.writer = writer; this.merge = merge; this.rateLimiter = new MergeRateLimiter(merge.getMergeProgress()); }
@Override public long pause(long bytes) throws MergePolicy.MergeAbortedException { totalBytesWritten.addAndGet(bytes); // While loop because we may wake up and check again when our rate limit // is changed while we were pausing: long paused = 0; long delta; while ((delta = maybePause(bytes, System.nanoTime())) >= 0) { // Keep waiting. paused += delta; } return paused; }
double curMBPerSec = merge.rateLimiter.getMBPerSec(); bytesToMB(merge.rateLimiter.totalBytesWritten), nsToSec(now - mergeStartNS), nsToSec(merge.rateLimiter.getTotalStoppedNS()), nsToSec(merge.rateLimiter.getTotalPausedNS()), rateToString(merge.rateLimiter.getMBPerSec()))); merge.rateLimiter.setMBPerSec(newMBPerSec);