/** * Register a metric with this sensor * @param metricName The name of the metric * @param stat The statistic to keep * @return true if metric is added to sensor, false if sensor is expired */ public boolean add(MetricName metricName, MeasurableStat stat) { return add(metricName, stat, null); }
private void recordPartitionLead(TopicPartition tp, long lead) { this.recordsFetchLead.record(lead); String name = partitionLeadMetricName(tp); Sensor recordsLead = this.metrics.getSensor(name); if (recordsLead == null) { Map<String, String> metricTags = new HashMap<>(2); metricTags.put("topic", tp.topic().replace('.', '_')); metricTags.put("partition", String.valueOf(tp.partition())); recordsLead = this.metrics.sensor(name); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLead, metricTags), new Value()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadMin, metricTags), new Min()); recordsLead.add(this.metrics.metricInstance(metricsRegistry.partitionRecordsLeadAvg, metricTags), new Avg()); } recordsLead.record(lead); }
public void close() { for (MetricName metricName : topLevelMetricNames) metrics.removeMetric(metricName); for (Sensor sensor : sensors) metrics.removeSensor(sensor.name()); } }
/** * Record an occurrence, this is just short-hand for {@link #record(double) record(1.0)} */ public void record() { if (shouldRecord()) { record(1.0); } }
private void checkForest(Set<Sensor> sensors) { if (!sensors.add(this)) throw new IllegalArgumentException("Circular dependency in sensors: " + name() + " is its own parent."); for (Sensor parent : parents) parent.checkForest(sensors); }
public void record(double value, long timeMs, boolean checkQuotas) { if (shouldRecord()) { this.lastRecordTime = timeMs; synchronized (this) { synchronized (metricLock()) { // increment all the stats for (Stat stat : this.stats) stat.record(config, value, timeMs); } if (checkQuotas) checkQuotas(timeMs); } for (Sensor parent : parents) parent.record(value, timeMs, checkQuotas); } }
/** * Record a value at a known time. This method is slightly faster than {@link #record(double)} since it will reuse * the time stamp. * @param value The value we are recording * @param timeMs The current POSIX time in milliseconds * @throws QuotaViolationException if recording this value moves a metric beyond its configured maximum or minimum * bound */ public void record(double value, long timeMs) { record(value, timeMs, true); }
public static void main(String[] args) { long iters = Long.parseLong(args[0]); Metrics metrics = new Metrics(); try { Sensor parent = metrics.sensor("parent"); Sensor child = metrics.sensor("child", parent); for (Sensor sensor : Arrays.asList(parent, child)) { sensor.add(metrics.metricName(sensor.name() + ".avg", "grp1"), new Avg()); sensor.add(metrics.metricName(sensor.name() + ".count", "grp1"), new Count()); sensor.add(metrics.metricName(sensor.name() + ".max", "grp1"), new Max()); sensor.add(new Percentiles(1024, 0.0, iters, BucketSizing.CONSTANT, new Percentile(metrics.metricName(sensor.name() + ".median", "grp1"), 50.0), new Percentile(metrics.metricName(sensor.name() + ".p_99", "grp1"), 99.0))); } long start = System.nanoTime(); for (int i = 0; i < iters; i++) parent.record(i); double ellapsed = (System.nanoTime() - start) / (double) iters; System.out.println(String.format("%.2f ns per metric recording.", ellapsed)); } finally { metrics.close(); } } }
/** * Verifies that concurrent sensor add, remove, updates and read don't result * in errors or deadlock. */ @Test public void testConcurrentReadUpdate() throws Exception { final Random random = new Random(); final Deque<Sensor> sensors = new ConcurrentLinkedDeque<>(); metrics = new Metrics(new MockTime(10)); SensorCreator sensorCreator = new SensorCreator(metrics); final AtomicBoolean alive = new AtomicBoolean(true); executorService = Executors.newSingleThreadExecutor(); executorService.submit(new ConcurrentMetricOperation(alive, "record", () -> sensors.forEach(sensor -> sensor.record(random.nextInt(10000))))); for (int i = 0; i < 10000; i++) { if (sensors.size() > 5) { Sensor sensor = random.nextBoolean() ? sensors.removeFirst() : sensors.removeLast(); metrics.removeSensor(sensor.name()); } StatType statType = StatType.forId(random.nextInt(StatType.values().length)); sensors.add(sensorCreator.createSensor(statType, i)); for (Sensor sensor : sensors) { for (KafkaMetric metric : sensor.metrics()) { assertNotNull("Invalid metric value", metric.metricValue()); } } } alive.set(false); }
@Test public void testHierarchicalSensors() { Sensor parent1 = metrics.sensor("test.parent1"); parent1.add(metrics.metricName("test.parent1.count", "grp1"), new Count()); Sensor parent2 = metrics.sensor("test.parent2"); parent2.add(metrics.metricName("test.parent2.count", "grp1"), new Count()); Sensor child1 = metrics.sensor("test.child1", parent1, parent2); child1.add(metrics.metricName("test.child1.count", "grp1"), new Count()); Sensor child2 = metrics.sensor("test.child2", parent1); child2.add(metrics.metricName("test.child2.count", "grp1"), new Count()); Sensor grandchild = metrics.sensor("test.grandchild", child1); grandchild.add(metrics.metricName("test.grandchild.count", "grp1"), new Count()); parent1.record(); parent2.record(); child1.record(); child2.record(); grandchild.record(); double p1 = (double) parent1.metrics().get(0).metricValue(); double p2 = (double) parent2.metrics().get(0).metricValue(); double c1 = (double) child1.metrics().get(0).metricValue(); double c2 = (double) child2.metrics().get(0).metricValue(); double gc = (double) grandchild.metrics().get(0).metricValue();
@Override public Throwable call() { try { assertTrue(latch.await(5, TimeUnit.SECONDS)); for (int j = 0; j != 20; ++j) { sensor.record(j * index, System.currentTimeMillis() + j, false); sensor.checkQuotas(); } return null; } catch (Throwable e) { return e; } } }));
@Test public void testExpiredSensor() { MetricConfig config = new MetricConfig(); Time mockTime = new MockTime(); Metrics metrics = new Metrics(config, Arrays.asList((MetricsReporter) new JmxReporter()), mockTime, true); long inactiveSensorExpirationTimeSeconds = 60L; Sensor sensor = new Sensor(metrics, "sensor", null, config, mockTime, inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO); assertTrue(sensor.add(metrics.metricName("test1", "grp1"), new Avg())); Map<String, String> emptyTags = Collections.emptyMap(); MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags); MetricName totalMetricName = new MetricName("total", "test", "", emptyTags); Meter meter = new Meter(rateMetricName, totalMetricName); assertTrue(sensor.add(meter)); mockTime.sleep(TimeUnit.SECONDS.toMillis(inactiveSensorExpirationTimeSeconds + 1)); assertFalse(sensor.add(metrics.metricName("test3", "grp1"), new Avg())); assertFalse(sensor.add(meter)); metrics.close(); }
/** * Remove a sensor (if it exists), associated metrics and its children. * * @param name The name of the sensor to be removed */ public void removeSensor(String name) { Sensor sensor = sensors.get(name); if (sensor != null) { List<Sensor> childSensors = null; synchronized (sensor) { synchronized (this) { if (sensors.remove(name, sensor)) { for (KafkaMetric metric : sensor.metrics()) removeMetric(metric.metricName()); log.debug("Removed sensor with name {}", name); childSensors = childrenSensors.remove(sensor); for (final Sensor parent : sensor.parents()) { childrenSensors.getOrDefault(parent, emptyList()).remove(sensor); } } } } if (childSensors != null) { for (Sensor childSensor : childSensors) removeSensor(childSensor.name()); } } }
@Test public void testIdempotentAdd() { final Metrics metrics = new Metrics(); final Sensor sensor = metrics.sensor("sensor"); assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Avg())); // adding the same metric to the same sensor is a no-op assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Avg())); // but adding the same metric to a DIFFERENT sensor is an error final Sensor anotherSensor = metrics.sensor("another-sensor"); try { anotherSensor.add(metrics.metricName("test-metric", "test-group"), new Avg()); fail("should have thrown"); } catch (final IllegalArgumentException ignored) { // pass } // note that adding a different metric with the same name is also a no-op assertTrue(sensor.add(metrics.metricName("test-metric", "test-group"), new Sum())); // so after all this, we still just have the original metric registered assertEquals(1, sensor.metrics().size()); assertEquals(org.apache.kafka.common.metrics.stats.Avg.class, sensor.metrics().get(0).measurable().getClass()); }
@Override public void flush() { if (flushTime.shouldRecord()) { measureLatency( () -> { inner.flush(); return null; }, flushTime); } else { inner.flush(); } }
@Test public void testShouldRecord() { MetricConfig debugConfig = new MetricConfig().recordLevel(Sensor.RecordingLevel.DEBUG); MetricConfig infoConfig = new MetricConfig().recordLevel(Sensor.RecordingLevel.INFO); Sensor infoSensor = new Sensor(null, "infoSensor", null, debugConfig, new SystemTime(), 0, Sensor.RecordingLevel.INFO); assertTrue(infoSensor.shouldRecord()); infoSensor = new Sensor(null, "infoSensor", null, debugConfig, new SystemTime(), 0, Sensor.RecordingLevel.DEBUG); assertTrue(infoSensor.shouldRecord()); Sensor debugSensor = new Sensor(null, "debugSensor", null, infoConfig, new SystemTime(), 0, Sensor.RecordingLevel.INFO); assertTrue(debugSensor.shouldRecord()); debugSensor = new Sensor(null, "debugSensor", null, infoConfig, new SystemTime(), 0, Sensor.RecordingLevel.DEBUG); assertFalse(debugSensor.shouldRecord()); }
/** * Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will * receive every value recorded with this sensor. * @param name The name of the sensor * @param config A default configuration to use for this sensor for metrics that don't have their own config * @param inactiveSensorExpirationTimeSeconds If no value if recorded on the Sensor for this duration of time, * it is eligible for removal * @param parents The parent sensors * @param recordingLevel The recording level. * @return The sensor that is created */ public synchronized Sensor sensor(String name, MetricConfig config, long inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel recordingLevel, Sensor... parents) { Sensor s = getSensor(name); if (s == null) { s = new Sensor(this, name, parents, config == null ? this.config : config, time, inactiveSensorExpirationTimeSeconds, recordingLevel); this.sensors.put(name, s); if (parents != null) { for (Sensor parent : parents) { List<Sensor> children = childrenSensors.get(parent); if (children == null) { children = new ArrayList<>(); childrenSensors.put(parent, children); } children.add(s); } } log.debug("Added sensor with name {}", name); } return s; }
/** * Check if we have violated our quota for any metric that has a configured quota */ public void checkQuotas() { checkQuotas(time.milliseconds()); }
Sensor(Metrics registry, String name, Sensor[] parents, MetricConfig config, Time time, long inactiveSensorExpirationTimeSeconds, RecordingLevel recordingLevel) { super(); this.registry = registry; this.name = Utils.notNull(name); this.parents = parents == null ? new Sensor[0] : parents; this.metrics = new LinkedHashMap<>(); this.stats = new ArrayList<>(); this.config = config; this.time = time; this.inactiveSensorExpirationTimeMs = TimeUnit.MILLISECONDS.convert(inactiveSensorExpirationTimeSeconds, TimeUnit.SECONDS); this.lastRecordTime = time.milliseconds(); this.recordingLevel = recordingLevel; this.metricLock = new Object(); checkForest(new HashSet<Sensor>()); }
void recordBatchSplit() { this.batchSplitSensor.record(); } }