private Map<Double, Long> getPercentiles(Histogram histogram) { Map<Double, Long> percentiles = new TreeMap<Double, Long>(); for (double targetPercentile : options.targetPercentiles()) { percentiles.put(targetPercentile, options.targetUnit().convert(histogram.getValueAtPercentile(targetPercentile), TimeUnit.NANOSECONDS)); } return percentiles; }
/** * This is called periodically from the StatusThread. There's a single * StatusThread per Client process. We optionally serialize the interval to * log on this opportunity. * * @see com.yahoo.ycsb.measurements.OneMeasurement#getSummary() */ @Override public String getSummary() { Histogram intervalHistogram = getIntervalHistogramAndAccumulate(); // we use the summary interval as the histogram file interval. if (histogramLogWriter != null) { histogramLogWriter.outputIntervalHistogram(intervalHistogram); } DecimalFormat d = new DecimalFormat("#.##"); return "[" + getName() + ": Count=" + intervalHistogram.getTotalCount() + ", Max=" + intervalHistogram.getMaxValue() + ", Min=" + intervalHistogram.getMinValue() + ", Avg=" + d.format(intervalHistogram.getMean()) + ", 90=" + d.format(intervalHistogram.getValueAtPercentile(90)) + ", 99=" + d.format(intervalHistogram.getValueAtPercentile(99)) + ", 99.9=" + d.format(intervalHistogram.getValueAtPercentile(99.9)) + ", 99.99=" + d.format(intervalHistogram.getValueAtPercentile(99.99)) + "]"; }
private static Histogram createResultHistogram(final List<Histogram> list, final long start, final long end) { final Histogram result = new Histogram(TimeUnit.SECONDS.toNanos(10), 3); result.setStartTimeStamp(start); result.setEndTimeStamp(end); for (final Histogram hist : list) { result.add(hist); } return result; }
@Override public Histogram copyCorrectedForCoordinatedOmission(final long expectedIntervalBetweenValueSamples) { Histogram copy = new Histogram(this); copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples); return copy; }
public void updateStats() { topicLoadHistogram = topicLoadTimeRecorder.getIntervalHistogram(topicLoadHistogram); this.elapsedIntervalMs = (TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicLoadRecordStartTime); topicLoadRecordStartTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); this.meanTopicLoadMs = topicLoadHistogram.getMean(); this.medianTopicLoadMs = topicLoadHistogram.getValueAtPercentile(50); this.topicLoad95Ms = topicLoadHistogram.getValueAtPercentile(95); this.topicLoad99Ms = topicLoadHistogram.getValueAtPercentile(99); this.topicLoad999Ms = topicLoadHistogram.getValueAtPercentile(99.9); this.topicsLoad9999Ms = topicLoadHistogram.getValueAtPercentile(99.99); this.topicLoadCounts = topicLoadHistogram.getTotalCount(); }
private CommandLatency getMetric(Histogram histogram) { Map<Double, Long> percentiles = getPercentiles(histogram); TimeUnit timeUnit = options.targetUnit(); return new CommandLatency(timeUnit.convert(histogram.getMinValue(), TimeUnit.NANOSECONDS), timeUnit.convert( histogram.getMaxValue(), TimeUnit.NANOSECONDS), percentiles); }
final CountDownLatch LATCH = new CountDownLatch(threadCount); for (int i = 0; i < threadCount; i++) { final Histogram serviceTmHist = new Histogram(TimeUnit.SECONDS.toNanos(10), 3); final Histogram responseTmHist = new Histogram(TimeUnit.SECONDS.toNanos(10), 3); serviceTmHistograms.add(serviceTmHist); responseTmHistograms.add(responseTmHist);
private void run() { long shortestObservableInterval = Long.MAX_VALUE; while (!Thread.currentThread().isInterrupted()) { try { long before = System.nanoTime(); TimeUnit.NANOSECONDS.sleep(sleepNanos); // attempt to allocate an object to capture any effects due to allocation stalls allocatedObject = new Long[] {before}; long after = System.nanoTime(); long delta = after - before; shortestObservableInterval = Math.min(shortestObservableInterval, delta); long pauseNanos = delta - shortestObservableInterval; synchronized (histogram) { histogram.recordValue(pauseNanos); totalPauseNanos += pauseNanos; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Throwable e) { LOG.warn(e, "Unexpected error"); } } }
private void run() throws IOException final Histogram histogram = new Histogram(TimeUnit.SECONDS.toNanos(10), 3); final ByteBuffer buffer = ByteBuffer.allocateDirect(Configuration.MTU_LENGTH_DEFAULT); histogram.recordValue(duration); histogram.reset(); System.gc(); LockSupport.parkNanos(1000 * 1000 * 1000);
rateFormat.format(consumeRate), throughputFormat.format(consumeThroughput), dec.format(currentBacklog / 1000.0), // dec.format(microsToMillis(stats.publishLatency.getMean())), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(50))), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99))), dec.format(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))), throughputFormat.format(microsToMillis(stats.publishLatency.getMaxValue()))); result.publishLatencyAvg.add(microsToMillis(stats.publishLatency.getMean())); result.publishLatency50pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(50))); result.publishLatency75pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(75))); result.publishLatency95pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(95))); result.publishLatency99pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99))); result.publishLatency999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.9))); result.publishLatency9999pct.add(microsToMillis(stats.publishLatency.getValueAtPercentile(99.99))); result.publishLatencyMax.add(microsToMillis(stats.publishLatency.getMaxValue())); result.endToEndLatencyAvg.add(microsToMillis(stats.endToEndLatency.getMean())); result.endToEndLatency50pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(50))); result.endToEndLatency75pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(75))); result.endToEndLatency95pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(95))); result.endToEndLatency99pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99))); result.endToEndLatency999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.9))); result.endToEndLatency9999pct.add(microsToMillis(stats.endToEndLatency.getValueAtPercentile(99.99))); result.endToEndLatencyMax.add(microsToMillis(stats.endToEndLatency.getMaxValue())); log.info( "----- Aggregated Pub Latency (ms) avg: {} - 50%: {} - 95%: {} - 99%: {} - 99.9%: {} - 99.99%: {} - Max: {}", dec.format(agg.publishLatency.getMean() / 1000.0),
TimeUnit.SECONDS.sleep(5); "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} ms - 95pct: {} ms - 99pct: {} ms - 99.9pct: {} ms - 99.99pct: {} ms", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0)); reportHistogram.reset(); TimeUnit.SECONDS.sleep(100);
@Override public PeriodStats getPeriodStats() { List<PeriodStats> individualStats = get(workers, "/period-stats", PeriodStats.class); PeriodStats stats = new PeriodStats(); individualStats.forEach(is -> { stats.messagesSent += is.messagesSent; stats.bytesSent += is.bytesSent; stats.messagesReceived += is.messagesReceived; stats.bytesReceived += is.bytesReceived; stats.totalMessagesSent += is.totalMessagesSent; stats.totalMessagesReceived += is.totalMessagesReceived; try { stats.publishLatency.add(Histogram.decodeFromCompressedByteBuffer( ByteBuffer.wrap(is.publishLatencyBytes), TimeUnit.SECONDS.toMicros(30))); stats.endToEndLatency.add(Histogram.decodeFromCompressedByteBuffer( ByteBuffer.wrap(is.endToEndLatencyBytes), TimeUnit.HOURS.toMicros(12))); } catch (ArrayIndexOutOfBoundsException | DataFormatException e) { throw new RuntimeException(e); } }); return stats; }
new Histogram(3) : ((Histogram) intervalHistogram).copy(); accumulatedRegularHistogram.reset(); accumulatedRegularHistogram.setAutoResize(true); new Histogram(3); accumulatedRegularHistogram.add((Histogram) intervalHistogram); ((DoubleHistogram) movingWindowSumHistogram).add((DoubleHistogram) intervalHistogram); } else { ((Histogram) movingWindowSumHistogram).add((Histogram) intervalHistogram); ((Histogram) movingWindowSumHistogram).subtract((Histogram) prevHist); ((Histogram) intervalHistogram).getTotalCount(), ((Histogram) intervalHistogram).getValueAtPercentile(50.0) / config.outputValueUnitRatio, ((Histogram) intervalHistogram).getValueAtPercentile(90.0) / config.outputValueUnitRatio, ((Histogram) intervalHistogram).getMaxValue() / config.outputValueUnitRatio, accumulatedRegularHistogram.getTotalCount(), accumulatedRegularHistogram.getValueAtPercentile(50.0) / config.outputValueUnitRatio, accumulatedRegularHistogram.getValueAtPercentile(90.0) / config.outputValueUnitRatio, accumulatedRegularHistogram.getValueAtPercentile(99.0) / config.outputValueUnitRatio, accumulatedRegularHistogram.getValueAtPercentile(99.9) / config.outputValueUnitRatio, accumulatedRegularHistogram.getValueAtPercentile(99.99) / config.outputValueUnitRatio, accumulatedRegularHistogram.getMaxValue() / config.outputValueUnitRatio );
private static void printAggregatedStats() { Histogram reportHistogram = cumulativeRecorder.getIntervalHistogram(); log.info( "Aggregated latency stats --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - 99.999pct: {} - Max: {}", dec.format(reportHistogram.getMean()), (long) reportHistogram.getValueAtPercentile(50), (long) reportHistogram.getValueAtPercentile(95), (long) reportHistogram.getValueAtPercentile(99), (long) reportHistogram.getValueAtPercentile(99.9), (long) reportHistogram.getValueAtPercentile(99.99), (long) reportHistogram.getValueAtPercentile(99.999), (long) reportHistogram.getMaxValue()); }
@UserAggregationResult public Map<String,Number> result() { long totalCount = values != null ? values.getTotalCount() : doubles.getTotalCount(); boolean empty = totalCount == 0; Map<String,Number> result = new LinkedHashMap<>(percentiles.size()+6); result.put("min",values != null ? (Number)values.getMinValue() : (Number)doubles.getMinValue()); result.put("minNonZero",values != null ? (Number)values.getMinNonZeroValue() : (Number)doubles.getMinNonZeroValue()); result.put("max",values != null ? (Number)values.getMaxValue() : (Number)doubles.getMaxValue()); result.put("total",totalCount); result.put("mean",values != null ? values.getMean() : doubles.getMean()); result.put("stdev",values != null ? values.getStdDeviation() : doubles.getStdDeviation()); for (Double percentile : percentiles) { if (percentile != null && !empty) { if (values != null) { result.put(percentile.toString(), values.getValueAtPercentile(percentile * 100D)); } else { result.put(percentile.toString(), doubles.getValueAtPercentile(percentile * 100D)); } } } return result; } }
mean = (int) underlying.getMean(); p0 = (int) underlying.getValueAtPercentile(0); p5 = (int) underlying.getValueAtPercentile(5); p10 = (int) underlying.getValueAtPercentile(10); p15 = (int) underlying.getValueAtPercentile(15); p20 = (int) underlying.getValueAtPercentile(20); p25 = (int) underlying.getValueAtPercentile(25); p30 = (int) underlying.getValueAtPercentile(30); p35 = (int) underlying.getValueAtPercentile(35); p40 = (int) underlying.getValueAtPercentile(40); p45 = (int) underlying.getValueAtPercentile(45); p50 = (int) underlying.getValueAtPercentile(50); p55 = (int) underlying.getValueAtPercentile(55); p60 = (int) underlying.getValueAtPercentile(60); p65 = (int) underlying.getValueAtPercentile(65); p70 = (int) underlying.getValueAtPercentile(70); p75 = (int) underlying.getValueAtPercentile(75); p80 = (int) underlying.getValueAtPercentile(80); p85 = (int) underlying.getValueAtPercentile(85); p90 = (int) underlying.getValueAtPercentile(90); p95 = (int) underlying.getValueAtPercentile(95); p99 = (int) underlying.getValueAtPercentile(99); p99_5 = (int) underlying.getValueAtPercentile(99.5); p99_9 = (int) underlying.getValueAtPercentile(99.9); p99_95 = (int) underlying.getValueAtPercentile(99.95); p99_99 = (int) underlying.getValueAtPercentile(99.99); p100 = (int) underlying.getValueAtPercentile(100); totalCount = underlying.getTotalCount();
/** * Returns the request latency at a given percentile. * * @param host the host (if this is relevant in the way percentiles are categorized). * @param statement the statement (if this is relevant in the way percentiles are categorized). * @param exception the exception (if this is relevant in the way percentiles are categorized). * @param percentile the percentile (for example, {@code 99.0} for the 99th percentile). * @return the latency (in milliseconds) at the given percentile, or a negative value if it's not * available yet. * @see #computeKey(Host, Statement, Exception) */ public long getLatencyAtPercentile( Host host, Statement statement, Exception exception, double percentile) { checkArgument( percentile >= 0.0 && percentile < 100, "percentile must be between 0.0 and 100 (was %s)", percentile); Histogram histogram = getLastIntervalHistogram(host, statement, exception); if (histogram == null || histogram.getTotalCount() < minRecordedValues) return -1; return histogram.getValueAtPercentile(percentile); }
public void recordValue(long val) { _histo.recordValue(val); }