/** * Record a cruise control metric value. * @param ccm the {@link CruiseControlMetric} to record. */ void recordCruiseControlMetric(CruiseControlMetric ccm) { RawMetricType rawMetricType = ccm.rawMetricType(); ValueHolder valueHolder = _rawMetricsByType.computeIfAbsent(rawMetricType, mt -> getValueHolderFor(rawMetricType)); valueHolder.recordValue(ccm.value(), ccm.time()); }
private void recordMetric(CruiseControlMetric ccm) { RawMetricType rawMetricType = ccm.rawMetricType(); switch (rawMetricType.metricScope()) { case BROKER: _brokerMetrics.recordCruiseControlMetric(ccm); break; case TOPIC: TopicMetric tm = (TopicMetric) ccm; _dotHandledTopicMetrics.computeIfAbsent(tm.topic(), t -> new RawMetricsHolder()) .recordCruiseControlMetric(ccm); break; case PARTITION: PartitionMetric pm = (PartitionMetric) ccm; _dotHandledPartitionMetrics.computeIfAbsent(new TopicPartition(pm.topic(), pm.partition()), tp -> new RawMetricsHolder()) .recordCruiseControlMetric(ccm); _dotHandledTopicsWithPartitionSizeReported.add(pm.topic()); break; default: throw new IllegalStateException(String.format("Should never be here. Unrecognized metric scope %s", rawMetricType.metricScope())); } }
@Test public void testMissingBrokerCpuUtilization() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); for (CruiseControlMetric metric : metrics) { if (metric.rawMetricType() == RawMetricType.BROKER_CPU_UTIL && metric.brokerId() == BROKER_ID_0) { // Do nothing and skip the metric. } else { processor.addMetric(metric); } } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals("Should have ignored partitions on broker 0", 1, samples.partitionMetricSamples().size()); assertEquals("Should have ignored broker 0", 1, samples.brokerMetricSamples().size()); }
@Test public void testMissingOtherBrokerMetrics() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); for (CruiseControlMetric metric : metrics) { if (metric.rawMetricType() == RawMetricType.BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX && metric.brokerId() == BROKER_ID_0) { // Do nothing and skip the metric. } else { processor.addMetric(metric); } } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals("Should have all 4 partition metrics.", 4, samples.partitionMetricSamples().size()); assertEquals("Should have ignored broker 0", 1, samples.brokerMetricSamples().size()); }
@Test public void testMissingPartitionSizeMetric() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); for (CruiseControlMetric metric : metrics) { boolean shouldAdd = true; if (metric.rawMetricType() == RawMetricType.PARTITION_SIZE) { PartitionMetric pm = (PartitionMetric) metric; if (pm.topic().equals(TOPIC1) && pm.partition() == P0) { shouldAdd = false; } } if (shouldAdd) { processor.addMetric(metric); } } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals("Should have ignored partition " + T1P0, 3, samples.partitionMetricSamples().size()); assertEquals("Should have reported both brokers", 2, samples.brokerMetricSamples().size()); }
@Test public void testMissingTopicBytesInMetric() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); Set<RawMetricType> metricTypeToExclude = new HashSet<>(Arrays.asList(TOPIC_BYTES_IN, TOPIC_BYTES_OUT, TOPIC_REPLICATION_BYTES_IN, TOPIC_REPLICATION_BYTES_OUT)); for (CruiseControlMetric metric : metrics) { if (metricTypeToExclude.contains(metric.rawMetricType())) { TopicMetric tm = (TopicMetric) metric; if (tm.brokerId() == BROKER_ID_0 && tm.topic().equals(TOPIC1)) { continue; } } processor.addMetric(metric); } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals(4, samples.partitionMetricSamples().size()); assertEquals(2, samples.brokerMetricSamples().size()); for (PartitionMetricSample sample : samples.partitionMetricSamples()) { if (sample.entity().tp().equals(T1P0)) { // T1P0 should not have any IO or CPU usage. validatePartitionMetricSample(sample, _time.milliseconds() + 2, 0.0, 0.0, 0.0, 100.0); } } }
records = consumer.poll(10); for (ConsumerRecord<String, CruiseControlMetric> record : records) { metricTypes.add((int) record.value().rawMetricType().id());
@Test public void testBrokerMetricSerde() throws UnknownVersionException { BrokerMetric brokerMetric = new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, 123L, 0, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(brokerMetric)); assertEquals(CruiseControlMetric.MetricClassId.BROKER_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.ALL_TOPIC_BYTES_IN.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(VALUE, deserialized.value(), 0.000001); }
@Test public void testTopicMetricSerde() throws UnknownVersionException { TopicMetric topicMetric = new TopicMetric(RawMetricType.TOPIC_BYTES_IN, 123L, 0, TOPIC, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(topicMetric)); assertEquals(CruiseControlMetric.MetricClassId.TOPIC_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.TOPIC_BYTES_IN.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(TOPIC, ((TopicMetric) deserialized).topic()); assertEquals(VALUE, deserialized.value(), 0.000001); }
@Test public void testPartitionMetricSerde() throws UnknownVersionException { PartitionMetric partitionMetric = new PartitionMetric(RawMetricType.PARTITION_SIZE, 123L, 0, TOPIC, PARTITION, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(partitionMetric)); assertEquals(CruiseControlMetric.MetricClassId.PARTITION_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.PARTITION_SIZE.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(TOPIC, ((PartitionMetric) deserialized).topic()); assertEquals(PARTITION, ((PartitionMetric) deserialized).partition()); assertEquals(VALUE, deserialized.value(), 0.000001); } }