/** * Register a metric with this sensor * @param metricName The name of the metric * @param stat The statistic to keep * @return true if metric is added to sensor, false if sensor is expired */ public boolean add(MetricName metricName, MeasurableStat stat) { return add(metricName, stat, null); }
/** * Register a compound statistic with this sensor with no config override * @param stat The stat to register * @return true if stat is added to sensor, false if sensor is expired */ public boolean add(CompoundStat stat) { return add(stat, null); }
public static Sensor throttleTimeSensor(SenderMetricsRegistry metrics) { Sensor produceThrottleTimeSensor = metrics.sensor("produce-throttle-time"); produceThrottleTimeSensor.add(metrics.produceThrottleTimeAvg, new Avg()); produceThrottleTimeSensor.add(metrics.produceThrottleTimeMax, new Max()); return produceThrottleTimeSensor; }
public static Sensor throttleTimeSensor(Metrics metrics, FetcherMetricsRegistry metricsRegistry) { Sensor fetchThrottleTimeSensor = metrics.sensor("fetch-throttle-time"); fetchThrottleTimeSensor.add(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg), new Avg()); fetchThrottleTimeSensor.add(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax), new Max()); return fetchThrottleTimeSensor; }
public SenderMetrics(SenderMetricsRegistry metrics, Metadata metadata, KafkaClient client, Time time) { this.metrics = metrics; this.time = time; this.batchSizeSensor = metrics.sensor("batch-size"); this.batchSizeSensor.add(metrics.batchSizeAvg, new Avg()); this.batchSizeSensor.add(metrics.batchSizeMax, new Max()); this.compressionRateSensor = metrics.sensor("compression-rate"); this.compressionRateSensor.add(metrics.compressionRateAvg, new Avg()); this.queueTimeSensor = metrics.sensor("queue-time"); this.queueTimeSensor.add(metrics.recordQueueTimeAvg, new Avg()); this.queueTimeSensor.add(metrics.recordQueueTimeMax, new Max()); this.requestTimeSensor = metrics.sensor("request-time"); this.requestTimeSensor.add(metrics.requestLatencyAvg, new Avg()); this.requestTimeSensor.add(metrics.requestLatencyMax, new Max()); this.recordsPerRequestSensor = metrics.sensor("records-per-request"); this.recordsPerRequestSensor.add(new Meter(metrics.recordSendRate, metrics.recordSendTotal)); this.recordsPerRequestSensor.add(metrics.recordsPerRequestAvg, new Avg()); this.retrySensor = metrics.sensor("record-retries"); this.retrySensor.add(new Meter(metrics.recordRetryRate, metrics.recordRetryTotal)); this.errorSensor = metrics.sensor("errors"); this.errorSensor.add(new Meter(metrics.recordErrorRate, metrics.recordErrorTotal)); this.maxRecordSizeSensor = metrics.sensor("record-size"); this.maxRecordSizeSensor.add(metrics.recordSizeMax, new Max()); this.maxRecordSizeSensor.add(metrics.recordSizeAvg, new Avg()); this.metrics.addMetric(metrics.requestsInFlight, (config, now) -> client.inFlightRequestCount()); this.metrics.addMetric(metrics.metadataAge, (config, now) -> (now - metadata.lastSuccessfulUpdate()) / 1000.0); this.batchSplitSensor = metrics.sensor("batch-split-rate"); this.batchSplitSensor.add(new Meter(metrics.batchSplitRate, metrics.batchSplitTotal)); }
private void recordPartitionLead(TopicPartition tp, long lead) { this.recordsFetchLead.record(lead); String name = partitionLeadMetricName(tp); Sensor recordsLead = this.metrics.getSensor(name); if (recordsLead == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLead = this.metrics.sensor(name); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLead, metricTags), new Value()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadMin, metricTags), new Min()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadAvg, metricTags), new Avg()); } recordsLead.record(lead); }
private void recordPartitionLag(TopicPartition tp, long lag) { this.recordsFetchLag.record(lag); String name = partitionLagMetricName(tp); Sensor recordsLag = this.metrics.getSensor(name); if (recordsLag == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLag = this.metrics.sensor(name); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLag, metricTags), new Value()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagMax, metricTags), new Max()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagAvg, metricTags), new Avg()); } recordsLag.record(lag); }
public void maybeRegisterConnectionMetrics(String connectionId) { if (!connectionId.isEmpty() && metricsPerConnection) { // if one sensor of the metrics has been registered for the connection, // then all other sensors should have been registered; and vice versa String nodeRequestName = "node-" + connectionId + ".bytes-sent"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest == null) { String metricGrpName = metricGrpPrefix + "-node-metrics"; Map<String, String> tags = new LinkedHashMap<>(metricTags); tags.put("node-id", "node-" + connectionId); nodeRequest = sensor(nodeRequestName); nodeRequest.add(createMeter(metrics, metricGrpName, tags, "outgoing-byte", "outgoing bytes")); nodeRequest.add(createMeter(metrics, metricGrpName, tags, new Count(), "request", "requests sent")); MetricName metricName = metrics.metricName("request-size-avg", metricGrpName, "The average size of requests sent.", tags); nodeRequest.add(metricName, new Avg()); metricName = metrics.metricName("request-size-max", metricGrpName, "The maximum size of any request sent.", tags); nodeRequest.add(metricName, new Max()); String nodeResponseName = "node-" + connectionId + ".bytes-received"; Sensor nodeResponse = sensor(nodeResponseName); nodeResponse.add(createMeter(metrics, metricGrpName, tags, "incoming-byte", "incoming bytes")); nodeResponse.add(createMeter(metrics, metricGrpName, tags, new Count(), "response", "responses received")); String nodeTimeName = "node-" + connectionId + ".latency"; Sensor nodeRequestTime = sensor(nodeTimeName); metricName = metrics.metricName("request-latency-avg", metricGrpName, tags); nodeRequestTime.add(metricName, new Avg()); metricName = metrics.metricName("request-latency-max", metricGrpName, tags); nodeRequestTime.add(metricName, new Max()); } } }
/** * Create a new buffer pool * * @param memory The maximum amount of memory that this buffer pool can allocate * @param poolableSize The buffer size to cache in the free list rather than deallocating * @param metrics instance of Metrics * @param time time instance * @param metricGrpName logical group name for metrics */ public BufferPool(long memory, int poolableSize, Metrics metrics, Time time, String metricGrpName) { this.poolableSize = poolableSize; this.lock = new ReentrantLock(); this.free = new ArrayDeque<>(); this.waiters = new ArrayDeque<>(); this.totalMemory = memory; this.nonPooledAvailableMemory = memory; this.metrics = metrics; this.time = time; this.waitTime = this.metrics.sensor(WAIT_TIME_SENSOR_NAME); MetricName rateMetricName = metrics.metricName("bufferpool-wait-ratio", metricGrpName, "The fraction of time an appender waits for space allocation."); MetricName totalMetricName = metrics.metricName("bufferpool-wait-time-total", metricGrpName, "The total time an appender waits for space allocation."); this.waitTime.add(new Meter(TimeUnit.NANOSECONDS, rateMetricName, totalMetricName)); }
private ConsumerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) { this.metricGrpName = metricGrpPrefix + "-coordinator-metrics"; this.commitLatency = metrics.sensor("commit-latency"); this.commitLatency.add(metrics.metricName("commit-latency-avg", this.metricGrpName, "The average time taken for a commit request"), new Avg()); this.commitLatency.add(metrics.metricName("commit-latency-max", this.metricGrpName, "The max time taken for a commit request"), new Max()); this.commitLatency.add(createMeter(metrics, metricGrpName, "commit", "commit calls")); Measurable numParts = new Measurable() { public double measure(MetricConfig config, long now) { // Get the number of assigned partitions in a thread safe manner return subscriptions.numAssignedPartitions(); } }; metrics.addMetric(metrics.metricName("assigned-partitions", this.metricGrpName, "The number of partitions currently assigned to this consumer"), numParts); } }
void addPartitionSensors(int partition) { Sensor recordsProducedSensor = metrics.sensor("records-produced-partition-" + partition); recordsProducedSensor.add(new MetricName("records-produced-rate-partition-" + partition, METRIC_GROUP_NAME, "The average number of records per second that are produced to this partition", _tags), new Rate()); _recordsProducedPerPartition.put(partition, recordsProducedSensor); Sensor errorsSensor = metrics.sensor("produce-error-partition-" + partition); errorsSensor.add(new MetricName("produce-error-rate-partition-" + partition, METRIC_GROUP_NAME, "The average number of errors per second when producing to this partition", _tags), new Rate()); _produceErrorPerPartition.put(partition, errorsSensor); } }
private FetchManagerMetrics(Metrics metrics, FetcherMetricsRegistry metricsRegistry) { this.metrics = metrics; this.metricsRegistry = metricsRegistry; this.bytesFetched = metrics.sensor("bytes-fetched"); this.bytesFetched.add(metrics.metricInstance(metricsRegistry.fetchSizeAvg), new Avg()); this.bytesFetched.add(metrics.metricInstance(metricsRegistry.fetchSizeMax), new Max()); this.bytesFetched.add(new Meter(metrics.metricInstance(metricsRegistry.bytesConsumedRate), metrics.metricInstance(metricsRegistry.bytesConsumedTotal))); this.recordsFetched = metrics.sensor("records-fetched"); this.recordsFetched.add(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg), new Avg()); this.recordsFetched.add(new Meter(metrics.metricInstance(metricsRegistry.recordsConsumedRate), metrics.metricInstance(metricsRegistry.recordsConsumedTotal))); this.fetchLatency = metrics.sensor("fetch-latency"); this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyAvg), new Avg()); this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyMax), new Max()); this.fetchLatency.add(new Meter(new Count(), metrics.metricInstance(metricsRegistry.fetchRequestRate), metrics.metricInstance(metricsRegistry.fetchRequestTotal))); this.recordsFetchLag = metrics.sensor("records-lag"); this.recordsFetchLag.add(metrics.metricInstance(metricsRegistry.recordsLagMax), new Max()); this.recordsFetchLead = metrics.sensor("records-lead"); this.recordsFetchLead.add(metrics.metricInstance(metricsRegistry.recordsLeadMin), new Min()); }
private void recordTopicFetchMetrics(String topic, int bytes, int records) { // record bytes fetched String name = "topic." + topic + ".bytes-fetched"; Sensor bytesFetched = this.metrics.getSensor(name); if (bytesFetched == null) { Map<String, String> metricTags = Collections.singletonMap("topic", topic.replace('.', '_')); bytesFetched = this.metrics.sensor(name); bytesFetched.add(this.metrics.metricInstance(metricsRegistry.topicFetchSizeAvg, metricTags), new Avg()); bytesFetched.add(this.metrics.metricInstance(metricsRegistry.topicFetchSizeMax, metricTags), new Max()); bytesFetched.add(new Meter(this.metrics.metricInstance(metricsRegistry.topicBytesConsumedRate, metricTags), this.metrics.metricInstance(metricsRegistry.topicBytesConsumedTotal, metricTags))); } bytesFetched.record(bytes); // record records fetched name = "topic." + topic + ".records-fetched"; Sensor recordsFetched = this.metrics.getSensor(name); if (recordsFetched == null) { Map<String, String> metricTags = new HashMap<>(1); metricTags.put("topic", topic.replace('.', '_')); recordsFetched = this.metrics.sensor(name); recordsFetched.add(this.metrics.metricInstance(metricsRegistry.topicRecordsPerRequestAvg, metricTags), new Avg()); recordsFetched.add(new Meter(this.metrics.metricInstance(metricsRegistry.topicRecordsConsumedRate, metricTags), this.metrics.metricInstance(metricsRegistry.topicRecordsConsumedTotal, metricTags))); } recordsFetched.record(records); }
@Before public void setup() throws Exception { metrics = new Metrics(); metrics.addReporter(new JmxReporter()); sensor = metrics.sensor("kafka.requests"); countMetricName = metrics.metricName("pack.bean1.count", "grp1"); sensor.add(countMetricName, new Count()); sumMetricName = metrics.metricName("pack.bean1.sum", "grp1"); sensor.add(sumMetricName, new Sum()); }
@Test public void testExpiredSensor() { MetricConfig config = new MetricConfig(); Time mockTime = new MockTime(); Metrics metrics = new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), mockTime, true); long inactiveSensorExpirationTimeSeconds = 60L; Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime, inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO); assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg())); Map<String, String> emptyTags = Collections.emptyMap(); MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags); MetricName totalMetricName = new MetricName("total", "test", "", emptyTags); Meter meter = new Meter(rateMetricName, totalMetricName); assertTrue(sensor.add(meter)); mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1)); assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg())); assertFalse(sensor.add(meter)); metrics.close(); }
public static void main(String[] args) { long iters = Long.parseLong(args[0]); Metrics metrics = new Metrics(); try { Sensor parent = metrics.sensor("parent"); Sensor child = metrics.sensor("child", parent); for (Sensor sensor : Arrays.asList(parent, child)) { sensor.add(metrics.metricName(sensor.name() + ".avg", "grp1"), new Avg()); sensor.add(metrics.metricName(sensor.name() + ".count", "grp1"), new Count()); sensor.add(metrics.metricName(sensor.name() + ".max", "grp1"), new Max()); sensor.add(new Percentiles(1024, 0.0, iters, BucketSizing.CONSTANT, new Percentile(metrics.metricName(sensor.name() + ".median", "grp1"), 50.0), new Percentile(metrics.metricName(sensor.name() + ".p_99", "grp1"), 99.0))); } long start = System.nanoTime(); for (int i = 0; i < iters; i++) parent.record(i); double ellapsed = (System.nanoTime() - start) / (double) iters; System.out.println(String.format("%.2f ns per metric recording.", ellapsed)); } finally { metrics.close(); } } }
@Test public void testIdempotentAdd() { final Metrics metrics = new Metrics(); final Sensor sensor = metrics.sensor("sensor"); assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Avg())); // adding the same metric to the same sensor is a no-op assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Avg())); // but adding the same metric to a DIFFERENT sensor is an error final Sensor anotherSensor = metrics.sensor("another-sensor"); try { anotherSensor.add(metrics.metricName("test-metric", "test-group"), new Avg()); fail("should have thrown"); } catch (final IllegalArgumentException ignored) { // pass } // note that adding a different metric with the same name is also a no-op assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Sum())); // so after all this, we still just have the original metric registered assertEquals(1, sensor.metrics().size()); assertEquals(org.apache.kafka.common.metrics.stats.Avg.class, sensor.metrics().get(0).measurable().getClass()); }
private void registerMetrics(Metrics metrics, String metricGrpName) { MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); Measurable waitingThreads = new Measurable() { public double measure(MetricConfig config, long now) { return free.queued(); } }; metrics.addMetric(metricName, waitingThreads); metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); Measurable totalBytes = new Measurable() { public double measure(MetricConfig config, long now) { return free.totalMemory(); } }; metrics.addMetric(metricName, totalBytes); metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); Measurable availableBytes = new Measurable() { public double measure(MetricConfig config, long now) { return free.availableMemory(); } }; metrics.addMetric(metricName, availableBytes); Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); MetricName rateMetricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); MetricName totalMetricName = metrics.metricName("buffer-exhausted-total", metricGrpName, "The total number of record sends that are dropped due to buffer exhaustion"); bufferExhaustedRecordSensor.add(new Meter(rateMetricName, totalMetricName)); }
@Test public void testQuotas() { Sensor sensor = metrics.sensor("test"); sensor.add(metrics.metricName("test1.total", "grp1"), new Total(), new MetricConfig().quota(Quota.upperBound(5.0))); sensor.add(metrics.metricName("test2.total", "grp1"), new Total(), new MetricConfig().quota(Quota.lowerBound(0.0))); sensor.record(5.0); try { sensor.record(1.0); fail("Should have gotten a quota violation."); } catch (QuotaViolationException e) { // this is good } assertEquals(6.0, (Double) metrics.metrics().get(metrics.metricName("test1.total", "grp1")).metricValue(), EPS); sensor.record(-6.0); try { sensor.record(-1.0); fail("Should have gotten a quota violation."); } catch (QuotaViolationException e) { // this is good } }