public MetricName topicRecordErrorRate(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordErrorRate, tags); }
/** topic level metrics **/ public MetricName topicRecordSendRate(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordSendRate, tags); }
public MetricName topicRecordErrorTotal(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordErrorTotal, tags); }
public MetricName topicRecordSendTotal(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordSendTotal, tags); }
public MetricName topicCompressionRate(Map<String, String> tags) { return this.metrics.metricInstance(this.topicCompressionRate, tags); }
public MetricName topicRecordRetryRate(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordRetryRate, tags); }
public MetricName topicByteRate(Map<String, String> tags) { return this.metrics.metricInstance(this.topicByteRate, tags); }
public MetricName topicByteTotal(Map<String, String> tags) { return this.metrics.metricInstance(this.topicByteTotal, tags); }
public MetricName topicRecordRetryTotal(Map<String, String> tags) { return this.metrics.metricInstance(this.topicRecordRetryTotal, tags); }
public MetricName metricInstance(MetricNameTemplate template, String... keyValue) { return metricInstance(template, getTags(keyValue)); }
private MetricName createMetricName(String name, String description) { return this.metrics.metricInstance(createTemplate(name, METRIC_GROUP_NAME, description, this.tags)); }
public static Sensor throttleTimeSensor(Metrics metrics, FetcherMetricsRegistry metricsRegistry) { Sensor fetchThrottleTimeSensor = metrics.sensor("fetch-throttle-time"); fetchThrottleTimeSensor.add(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg), new Avg()); fetchThrottleTimeSensor.add(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax), new Max()); return fetchThrottleTimeSensor; }
private FetchManagerMetrics(Metrics metrics, FetcherMetricsRegistry metricsRegistry) { this.metrics = metrics; this.metricsRegistry = metricsRegistry; this.bytesFetched = metrics.sensor("bytes-fetched"); this.bytesFetched.add(metrics.metricInstance(metricsRegistry.fetchSizeAvg), new Avg()); this.bytesFetched.add(metrics.metricInstance(metricsRegistry.fetchSizeMax), new Max()); this.bytesFetched.add(new Meter(metrics.metricInstance(metricsRegistry.bytesConsumedRate), metrics.metricInstance(metricsRegistry.bytesConsumedTotal))); this.recordsFetched = metrics.sensor("records-fetched"); this.recordsFetched.add(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg), new Avg()); this.recordsFetched.add(new Meter(metrics.metricInstance(metricsRegistry.recordsConsumedRate), metrics.metricInstance(metricsRegistry.recordsConsumedTotal))); this.fetchLatency = metrics.sensor("fetch-latency"); this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyAvg), new Avg()); this.fetchLatency.add(metrics.metricInstance(metricsRegistry.fetchLatencyMax), new Max()); this.fetchLatency.add(new Meter(new Count(), metrics.metricInstance(metricsRegistry.fetchRequestRate), metrics.metricInstance(metricsRegistry.fetchRequestTotal))); this.recordsFetchLag = metrics.sensor("records-lag"); this.recordsFetchLag.add(metrics.metricInstance(metricsRegistry.recordsLagMax), new Max()); this.recordsFetchLead = metrics.sensor("records-lead"); this.recordsFetchLead.add(metrics.metricInstance(metricsRegistry.recordsLeadMin), new Min()); }
private void recordPartitionLead(TopicPartition tp, long lead) { this.recordsFetchLead.record(lead); String name = partitionLeadMetricName(tp); Sensor recordsLead = this.metrics.getSensor(name); if (recordsLead == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLead = this.metrics.sensor(name); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLead, metricTags), new Value()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadMin, metricTags), new Min()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadAvg, metricTags), new Avg()); } recordsLead.record(lead); }
private void recordPartitionLag(TopicPartition tp, long lag) { this.recordsFetchLag.record(lag); String name = partitionLagMetricName(tp); Sensor recordsLag = this.metrics.getSensor(name); if (recordsLag == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLag = this.metrics.sensor(name); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLag, metricTags), new Value()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagMax, metricTags), new Max()); recordsLag.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLagAvg, metricTags), new Avg()); } recordsLag.record(lag); }
@Test public void testFetchResponseMetricsPartialResponse() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); Map<MetricName, KafkaMetric> allMetrics = metrics.metrics(); KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L); for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); int expectedBytes = 0; for (Record record : records.records()) { if (record.offset() >= 1) expectedBytes += record.sizeInBytes(); } fetchRecords(tp0, records, Errors.NONE, 100L, 0); assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); assertEquals(2, (Double) recordsCountAverage.metricValue(), EPSILON); }
KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); assertEquals(6, (Double) recordsCountAverage.metricValue(), EPSILON);
@Test public void testFetchResponseMetricsWithOnePartitionError() { subscriptions.assignFromUser(Utils.mkSet(tp0, tp1)); subscriptions.seek(tp0, 0); subscriptions.seek(tp1, 0); Map<MetricName, KafkaMetric> allMetrics = metrics.metrics(); KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L); for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); Map<TopicPartition, FetchResponse.PartitionData<MemoryRecords>> partitions = new HashMap<>(); partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, MemoryRecords.EMPTY)); assertEquals(1, fetcher.sendFetches()); client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); consumerClient.poll(time.timer(0)); fetcher.fetchedRecords(); int expectedBytes = 0; for (Record record : records.records()) expectedBytes += record.sizeInBytes(); assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON); }
@Test public void testFetcherMetrics() { subscriptions.assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); Map<String, String> tags = new HashMap<>(); tags.put("topic", tp0.topic()); tags.put("partition", String.valueOf(tp0.partition())); MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); Map<MetricName, KafkaMetric> allMetrics = metrics.metrics(); KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); // recordsFetchLagMax should be initialized to NaN assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); // recordsFetchLagMax should be hw - fetchOffset after receiving an empty FetchResponse fetchRecords(tp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 0); assertEquals(100, (Double) recordsFetchLagMax.metricValue(), EPSILON); KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); assertEquals(100, (Double) partitionLag.metricValue(), EPSILON); // recordsFetchLagMax should be hw - offset of the last message after receiving a non-empty FetchResponse MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L); for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); fetchRecords(tp0, builder.build(), Errors.NONE, 200L, 0); assertEquals(197, (Double) recordsFetchLagMax.metricValue(), EPSILON); assertEquals(197, (Double) partitionLag.metricValue(), EPSILON); // verify de-registration of partition lag subscriptions.unsubscribe(); assertFalse(allMetrics.containsKey(partitionLagMetric)); }
subscriptions.seek(tp0, 0); MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin); Map<String, String> tags = new HashMap<>(2); tags.put("topic", tp0.topic());