public static void main(String[] args) { Map<String, String> metricTags = Collections.singletonMap("client-id", "client-id"); MetricConfig metricConfig = new MetricConfig().tags(metricTags); Metrics metrics = new Metrics(metricConfig); ProducerMetrics metricsRegistry = new ProducerMetrics(metrics); System.out.println(Metrics.toHtmlTable("kafka.producer", metricsRegistry.getAllTemplates())); }
private void registerMetrics(Metrics metrics, String metricGrpName) { MetricName metricName = metrics.metricName("waiting-threads", metricGrpName, "The number of user threads blocked waiting for buffer memory to enqueue their records"); Measurable waitingThreads = new Measurable() { public double measure(MetricConfig config, long now) { return free.queued(); } }; metrics.addMetric(metricName, waitingThreads); metricName = metrics.metricName("buffer-total-bytes", metricGrpName, "The maximum amount of buffer memory the client can use (whether or not it is currently used)."); Measurable totalBytes = new Measurable() { public double measure(MetricConfig config, long now) { return free.totalMemory(); } }; metrics.addMetric(metricName, totalBytes); metricName = metrics.metricName("buffer-available-bytes", metricGrpName, "The total amount of buffer memory that is not being used (either unallocated or in the free list)."); Measurable availableBytes = new Measurable() { public double measure(MetricConfig config, long now) { return free.availableMemory(); } }; metrics.addMetric(metricName, availableBytes); Sensor bufferExhaustedRecordSensor = metrics.sensor("buffer-exhausted-records"); MetricName rateMetricName = metrics.metricName("buffer-exhausted-rate", metricGrpName, "The average per-second number of record sends that are dropped due to buffer exhaustion"); MetricName totalMetricName = metrics.metricName("buffer-exhausted-total", metricGrpName, "The total number of record sends that are dropped due to buffer exhaustion"); bufferExhaustedRecordSensor.add(new Meter(rateMetricName, totalMetricName)); }
/** * Get the full set of internal metrics maintained by the producer. */ @Override public Map<MetricName, ? extends Metric> metrics() { return Collections.unmodifiableMap(this.metrics.metrics()); }
/** * Create a MetricName with the given name, group, description, and keyValue as tags, plus default tags specified in the metric * configuration. Tag in keyValue takes precedence if the same tag key is specified in the default metric configuration. * * @param name The name of the metric * @param group logical group name of the metrics to which this metric belongs * @param description A human-readable description to include in the metric * @param keyValue additional key/value attributes of the metric (must come in pairs) */ public MetricName metricName(String name, String group, String description, String... keyValue) { return metricName(name, group, description, getTags(keyValue)); }
public void close() { for (MetricName metricName : topLevelMetricNames) metrics.removeMetric(metricName); for (Sensor sensor : sensors) metrics.removeSensor(sensor.name()); } }
@Test public void testJmxRegistrationSanitization() throws Exception { Metrics metrics = new Metrics(); MBeanServer server = ManagementFactory.getPlatformMBeanServer(); try { metrics.addReporter(new JmxReporter()); Sensor sensor = metrics.sensor("kafka.requests"); sensor.add(metrics.metricName("name", "group", "desc", "id", "foo*"), new Total()); sensor.add(metrics.metricName("name", "group", "desc", "id", "foo+"), new Total()); sensor.add(metrics.metricName("name", "group", "desc", "id", "foo?"), new Total()); sensor.add(metrics.metricName("name", "group", "desc", "id", "foo:"), new Total()); sensor.add(metrics.metricName("name", "group", "desc", "id", "foo%"), new Total()); assertEquals(0.0, server.getAttribute(new ObjectName(":type=group,id=foo%"), "name")); metrics.removeMetric(metrics.metricName("name", "group", "desc", "id", "foo*")); metrics.removeMetric(metrics.metricName("name", "group", "desc", "id", "foo+")); metrics.removeMetric(metrics.metricName("name", "group", "desc", "id", "foo?")); metrics.removeMetric(metrics.metricName("name", "group", "desc", "id", "foo:")); metrics.removeMetric(metrics.metricName("name", "group", "desc", "id", "foo%")); assertFalse(server.isRegistered(new ObjectName(":type=group,id=foo%"))); } finally { metrics.close();
@Test public void testRemoveSensor() { int size = metrics.metrics().size(); Sensor parent1 = metrics.sensor("test.parent1"); parent1.add(metrics.metricName("test.parent1.count", "grp1"), new Count()); Sensor parent2 = metrics.sensor("test.parent2"); parent2.add(metrics.metricName("test.parent2.count", "grp1"), new Count()); Sensor child1 = metrics.sensor("test.child1", parent1, parent2); child1.add(metrics.metricName("test.child1.count", "grp1"), new Count()); Sensor child2 = metrics.sensor("test.child2", parent2); child2.add(metrics.metricName("test.child2.count", "grp1"), new Count()); Sensor grandChild1 = metrics.sensor("test.gchild2", child2); grandChild1.add(metrics.metricName("test.gchild2.count", "grp1"), new Count()); Sensor sensor = metrics.getSensor("test.parent1"); assertNotNull(sensor); metrics.removeSensor("test.parent1"); assertNull(metrics.getSensor("test.parent1")); assertNull(metrics.metrics().get(metrics.metricName("test.parent1.count", "grp1"))); assertNull(metrics.getSensor("test.child1")); assertNull(metrics.childrenSensors().get(sensor)); assertNull(metrics.metrics().get(metrics.metricName("test.child1.count", "grp1"))); sensor = metrics.getSensor("test.gchild2"); assertNotNull(sensor); metrics.removeSensor("test.gchild2"); assertNull(metrics.getSensor("test.gchild2")); assertNull(metrics.childrenSensors().get(sensor)); assertNull(metrics.metrics().get(metrics.metricName("test.gchild2.count", "grp1")));
private Selector createSelector(Map<String, Object> sslClientConfigs, final Integer netReadBufSize, final Integer netWriteBufSize, final Integer appBufSize) { TestSslChannelBuilder channelBuilder = new TestSslChannelBuilder(Mode.CLIENT); channelBuilder.configureBufferSizes(netReadBufSize, netWriteBufSize, appBufSize); this.channelBuilder = channelBuilder; this.channelBuilder.configure(sslClientConfigs); this.selector = new Selector(5000, new Metrics(), time, "MetricGroup", channelBuilder, new LogContext()); return selector; }
@Test public void testSenderMetricsTemplates() throws Exception { metrics.close(); Map<String, String> clientTags = Collections.singletonMap("client-id", "clientA"); metrics = new Metrics(new MetricConfig().tags(clientTags)); SenderMetricsRegistry metricsRegistry = new SenderMetricsRegistry(metrics); Sender sender = new Sender(logContext, client, metadata, this.accumulator, false, MAX_REQUEST_SIZE, ACKS_ALL, 1, metricsRegistry, time, REQUEST_TIMEOUT, 50, null, apiVersions); // Append a message so that topic metrics are created accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT); sender.run(time.milliseconds()); // connect sender.run(time.milliseconds()); // send produce request client.respond(produceResponse(tp0, 0, Errors.NONE, 0)); sender.run(time.milliseconds()); // Create throttle time metrics Sender.throttleTimeSensor(metricsRegistry); // Verify that all metrics except metrics-count have registered templates Set<MetricNameTemplate> allMetrics = new HashSet<>(); for (MetricName n : metrics.metrics().keySet()) { if (!n.group().equals("kafka-metrics-count")) allMetrics.add(new MetricNameTemplate(n.name(), n.group(), "", n.tags().keySet())); } TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.allTemplates()), "metrics", "templates"); }
@Before public void setup() throws Exception { metrics = new Metrics(); metrics.addReporter(new JmxReporter()); sensor = metrics.sensor("kafka.requests"); countMetricName = metrics.metricName("pack.bean1.count", "grp1"); sensor.add(countMetricName, new Count()); sumMetricName = metrics.metricName("pack.bean1.sum", "grp1"); sensor.add(sumMetricName, new Sum()); }
metrics.addMetric(metrics.metricName("direct.measurable", "grp1", "The fraction of time an appender waits for space allocation."), measurable); Sensor s = metrics.sensor("test.sensor"); s.add(metrics.metricName("test.avg", "grp1"), new Avg()); s.add(metrics.metricName("test.max", "grp1"), new Max()); s.add(metrics.metricName("test.min", "grp1"), new Min()); s.add(new Meter(TimeUnit.SECONDS, metrics.metricName("test.rate", "grp1"), metrics.metricName("test.total", "grp1"))); s.add(new Meter(TimeUnit.SECONDS, new Count(), metrics.metricName("test.occurences", "grp1"), metrics.metricName("test.occurences.total", "grp1"))); s.add(metrics.metricName("test.count", "grp1"), new Count()); s.add(new Percentiles(100, -100, 100, BucketSizing.CONSTANT, new Percentile(metrics.metricName("test.median", "grp1"), 50.0), new Percentile(metrics.metricName("test.perc99_9", "grp1"), 99.9))); Sensor s2 = metrics.sensor("test.sensor2"); s2.add(metrics.metricName("s2.total", "grp1"), new Total()); s2.record(5.0); metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.occurences", "grp1"))), EPS); assertEquals("s2 reflects the constant value", 5.0, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("s2.total", "grp1"))), EPS); assertEquals("Avg(0...9) = 4.5", 4.5, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.avg", "grp1"))), EPS); assertEquals("Max(0...9) = 9", count - 1, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.max", "grp1"))), EPS); assertEquals("Min(0...9) = 0", 0.0, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.min", "grp1"))), EPS); assertEquals("Rate(0...9) = 1.40625", sum / elapsedSecs, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.rate", "grp1"))), EPS); assertEquals(String.format("Occurrences(0...%d) = %f", count, count / elapsedSecs), count / elapsedSecs, metricValueFunc.apply(metrics.metrics().get(metrics.metricName("test.occurences", "grp1"))), EPS);
@Test public void testRemoveMetric() { int size = metrics.metrics().size(); metrics.addMetric(metrics.metricName("test1", "grp1"), new Count()); metrics.addMetric(metrics.metricName("test2", "grp1"), new Count()); assertNotNull(metrics.removeMetric(metrics.metricName("test1", "grp1"))); assertNull(metrics.metrics().get(metrics.metricName("test1", "grp1"))); assertNotNull(metrics.metrics().get(metrics.metricName("test2", "grp1"))); assertNotNull(metrics.removeMetric(metrics.metricName("test2", "grp1"))); assertNull(metrics.metrics().get(metrics.metricName("test2", "grp1"))); assertEquals(size, metrics.metrics().size()); }
@Test public void testRemoveInactiveMetrics() { Sensor s1 = metrics.sensor("test.s1", null, 1); s1.add(metrics.metricName("test.s1.count", "grp1"), new Count()); Sensor s2 = metrics.sensor("test.s2", null, 3); s2.add(metrics.metricName("test.s2.count", "grp1"), new Count()); assertNotNull("Sensor test.s1 must be present", metrics.getSensor("test.s1")); assertNotNull("MetricName test.s1.count must be present", metrics.metrics().get(metrics.metricName("test.s1.count", "grp1"))); assertNotNull("Sensor test.s2 must be present", metrics.getSensor("test.s2")); assertNotNull("MetricName test.s2.count must be present", metrics.metrics().get(metrics.metricName("test.s2.count", "grp1"))); assertNull("Sensor test.s1 should have been purged", metrics.getSensor("test.s1")); assertNull("MetricName test.s1.count should have been purged", metrics.metrics().get(metrics.metricName("test.s1.count", "grp1"))); assertNotNull("Sensor test.s2 must be present", metrics.getSensor("test.s2")); assertNotNull("MetricName test.s2.count must be present", metrics.metrics().get(metrics.metricName("test.s2.count", "grp1"))); time.sleep(2000); purger.run(); assertNotNull("Sensor test.s2 must be present", metrics.getSensor("test.s2")); assertNotNull("MetricName test.s2.count must be present", metrics.metrics().get(metrics.metricName("test.s2.count", "grp1"))); assertNull("Sensor test.s2 should have been purged", metrics.getSensor("test.s1")); assertNull("MetricName test.s2.count should have been purged",
@Test public void testRemoveChildSensor() { final Metrics metrics = new Metrics(); final Sensor parent = metrics.sensor("parent"); final Sensor child = metrics.sensor("child", parent); assertEquals(singletonList(child), metrics.childrenSensors().get(parent)); metrics.removeSensor("child"); assertEquals(emptyList(), metrics.childrenSensors().get(parent)); }
List<MetricsReporter> reporters = new ArrayList<>(); reporters.add(new JmxReporter(JMX_PREFIX)); Metrics metrics = new Metrics(new MetricConfig(), reporters, new SystemTime()); metrics.addMetric(metrics.metricName("offline-runnable-count", METRIC_GROUP_NAME, "The number of Service/App that are not fully running"), new Measurable() { @Override
@Test public void testFetcherMetrics() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); Map<String, String> tags = new HashMap<>(); tags.put("topic", tp0.topic()); tags.put("partition", String.valueOf(tp0.partition())); MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); Map<MetricName, KafkaMetric> allMetrics = metrics.metrics(); KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); // recordsFetchLagMax should be initialized to NaN assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); // recordsFetchLagMax should be hw - fetchOffset after receiving an empty FetchResponse fetchRecords(tp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 0); assertEquals(100, (Double) recordsFetchLagMax.metricValue(), EPSILON); KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); assertEquals(100, (Double) partitionLag.metricValue(), EPSILON); // recordsFetchLagMax should be hw - offset of the last message after receiving a non-empty FetchResponse MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L); for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); fetchRecords(tp0, builder.build(), Errors.NONE, 200L, 0); assertEquals(197, (Double) recordsFetchLagMax.metricValue(), EPSILON); assertEquals(197, (Double) partitionLag.metricValue(), EPSILON); // verify de-registration of partition lag subscriptions.unsubscribe(); assertFalse(allMetrics.containsKey(partitionLagMetric)); }
private void addSensor( final String key, final String metricNameString, final MeasurableStat stat, final List<TopicSensors.SensorMetric<ProducerRecord>> results ) { final String name = "prod-" + key + "-" + metricNameString + "-" + id; final MetricName metricName = new MetricName( metricNameString, "producer-metrics", "producer-" + name, ImmutableMap.of("key", key, "id", id) ); final Sensor existingSensor = metrics.getSensor(name); final Sensor sensor = metrics.sensor(name); // either a new sensor or a new metric with different id if (existingSensor == null || metrics.metrics().get(metricName) == null) { sensor.add(metricName, stat); } final KafkaMetric metric = metrics.metrics().get(metricName); results.add(new TopicSensors.SensorMetric<ProducerRecord>(sensor, metric, time, false) { void record(final ProducerRecord record) { sensor.record(1); super.record(record); } }); }
private void recordTopicFetchMetrics(String topic, int bytes, int records) { // record bytes fetched String name = "topic." + topic + ".bytes-fetched"; Sensor bytesFetched = this.metrics.getSensor(name); if (bytesFetched == null) { Map<String, String> metricTags = Collections.singletonMap("topic", topic.replace('.', '_')); bytesFetched = this.metrics.sensor(name); bytesFetched.add(this.metrics.metricInstance(metricsRegistry.topicFetchSizeAvg, metricTags), new Avg()); bytesFetched.add(this.metrics.metricInstance(metricsRegistry.topicFetchSizeMax, metricTags), new Max()); bytesFetched.add(new Meter(this.metrics.metricInstance(metricsRegistry.topicBytesConsumedRate, metricTags), this.metrics.metricInstance(metricsRegistry.topicBytesConsumedTotal, metricTags))); } bytesFetched.record(bytes); // record records fetched name = "topic." + topic + ".records-fetched"; Sensor recordsFetched = this.metrics.getSensor(name); if (recordsFetched == null) { Map<String, String> metricTags = new HashMap<>(1); metricTags.put("topic", topic.replace('.', '_')); recordsFetched = this.metrics.sensor(name); recordsFetched.add(this.metrics.metricInstance(metricsRegistry.topicRecordsPerRequestAvg, metricTags), new Avg()); recordsFetched.add(new Meter(this.metrics.metricInstance(metricsRegistry.topicRecordsConsumedRate, metricTags), this.metrics.metricInstance(metricsRegistry.topicRecordsConsumedTotal, metricTags))); } recordsFetched.record(records); }
@Test public void testExpiredSensor() { MetricConfig config = new MetricConfig(); Time mockTime = new MockTime(); Metrics metrics = new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), mockTime, true); long inactiveSensorExpirationTimeSeconds = 60L; Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime, inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO); assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg())); Map<String, String> emptyTags = Collections.emptyMap(); MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags); MetricName totalMetricName = new MetricName("total", "test", "", emptyTags); Meter meter = new Meter(rateMetricName, totalMetricName); assertTrue(sensor.add(meter)); mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1)); assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg())); assertFalse(sensor.add(meter)); metrics.close(); }
@Test public void testQuotas() { Sensor sensor = metrics.sensor("test"); sensor.add(metrics.metricName("test1.total", "grp1"), new Total(), new MetricConfig().quota(Quota.upperBound(5.0))); sensor.add(metrics.metricName("test2.total", "grp1"), new Total(), new MetricConfig().quota(Quota.lowerBound(0.0))); sensor.record(5.0); try { sensor.record(1.0); fail("Should have gotten a quota violation."); } catch (QuotaViolationException e) { // this is good } assertEquals(6.0, (Double) metrics.metrics().get(metrics.metricName("test1.total", "grp1")).metricValue(), EPS); sensor.record(-6.0); try { sensor.record(-1.0); fail("Should have gotten a quota violation."); } catch (QuotaViolationException e) { // this is good } }