@Override public String toString() { return String.format("[%s,%s,time=%d,brokerId=%d,topic=%s,value=%.3f]", MetricClassId.TOPIC_METRIC, rawMetricType(), time(), brokerId(), topic(), value()); } }
static TopicMetric fromBuffer(ByteBuffer buffer) throws UnknownVersionException { byte version = buffer.get(); if (version > METRIC_VERSION) { throw new UnknownVersionException("Cannot deserialize the topic metrics for version " + version + ". " + "Current version is " + METRIC_VERSION); } RawMetricType rawMetricType = RawMetricType.forId(buffer.get()); long time = buffer.getLong(); int brokerId = buffer.getInt(); int topicLength = buffer.getInt(); String topic = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), topicLength, StandardCharsets.UTF_8); buffer.position(buffer.position() + topicLength); double value = buffer.getDouble(); return new TopicMetric(rawMetricType, time, brokerId, topic, value); }
public static CruiseControlMetric fromBytes(byte[] bytes) throws UnknownVersionException { ByteBuffer buffer = ByteBuffer.wrap(bytes); switch (CruiseControlMetric.MetricClassId.forId(buffer.get())) { case BROKER_METRIC: return BrokerMetric.fromBuffer(buffer); case TOPIC_METRIC: return TopicMetric.fromBuffer(buffer); case PARTITION_METRIC: return PartitionMetric.fromBuffer(buffer); default: // This could happen when a new type of metric is added but we are still running the old code. // simply ignore the metric by returning a null. return null; } }
@Test public void testTopicMetricSerde() throws UnknownVersionException { TopicMetric topicMetric = new TopicMetric(RawMetricType.TOPIC_BYTES_IN, 123L, 0, TOPIC, 0.1); CruiseControlMetric deserialized = MetricSerde.fromBytes(MetricSerde.toBytes(topicMetric)); assertEquals(CruiseControlMetric.MetricClassId.TOPIC_METRIC.id(), deserialized.metricClassId().id()); assertEquals(RawMetricType.TOPIC_BYTES_IN.id(), deserialized.rawMetricType().id()); assertEquals(TIME, deserialized.time()); assertEquals(BROKER_ID, deserialized.brokerId()); assertEquals(TOPIC, ((TopicMetric) deserialized).topic()); assertEquals(VALUE, deserialized.value(), 0.000001); }
@Test public void testMissingTopicBytesInMetric() throws UnknownVersionException { CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(); Set<CruiseControlMetric> metrics = getCruiseControlMetrics(); Set<RawMetricType> metricTypeToExclude = new HashSet<>(Arrays.asList(TOPIC_BYTES_IN, TOPIC_BYTES_OUT, TOPIC_REPLICATION_BYTES_IN, TOPIC_REPLICATION_BYTES_OUT)); for (CruiseControlMetric metric : metrics) { if (metricTypeToExclude.contains(metric.rawMetricType())) { TopicMetric tm = (TopicMetric) metric; if (tm.brokerId() == BROKER_ID_0 && tm.topic().equals(TOPIC1)) { continue; } } processor.addMetric(metric); } MetricSampler.Samples samples = processor.process(getCluster(), Arrays.asList(T1P0, T1P1, T2P0, T2P1), MetricSampler.SamplingMode.ALL); assertEquals(4, samples.partitionMetricSamples().size()); assertEquals(2, samples.brokerMetricSamples().size()); for (PartitionMetricSample sample : samples.partitionMetricSamples()) { if (sample.entity().tp().equals(T1P0)) { // T1P0 should not have any IO or CPU usage. validatePartitionMetricSample(sample, _time.milliseconds() + 2, 0.0, 0.0, 0.0, 100.0); } } }
/** * Send a CruiseControlMetric to the Kafka topic. * @param ccm the Cruise Control metric to send. */ public void sendCruiseControlMetric(CruiseControlMetric ccm) { // Use topic name as key if existing so that the same sampler will be able to collect all the information // of a topic. String key = ccm.metricClassId() == CruiseControlMetric.MetricClassId.TOPIC_METRIC ? ((TopicMetric) ccm).topic() : Integer.toString(ccm.brokerId()); ProducerRecord<String, CruiseControlMetric> producerRecord = new ProducerRecord<>(_cruiseControlMetricsTopic, null, ccm.time(), key, ccm); LOG.debug("Sending Cruise Control metric {}.", ccm); _producer.send(producerRecord, new Callback() { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { LOG.warn("Failed to send Cruise Control metric {}", ccm); _numMetricSendFailure++; } } }); }
private void recordMetric(CruiseControlMetric ccm) { RawMetricType rawMetricType = ccm.rawMetricType(); switch (rawMetricType.metricScope()) { case BROKER: _brokerMetrics.recordCruiseControlMetric(ccm); break; case TOPIC: TopicMetric tm = (TopicMetric) ccm; _dotHandledTopicMetrics.computeIfAbsent(tm.topic(), t -> new RawMetricsHolder()) .recordCruiseControlMetric(ccm); break; case PARTITION: PartitionMetric pm = (PartitionMetric) ccm; _dotHandledPartitionMetrics.computeIfAbsent(new TopicPartition(pm.topic(), pm.partition()), tp -> new RawMetricsHolder()) .recordCruiseControlMetric(ccm); _dotHandledTopicsWithPartitionSizeReported.add(pm.topic()); break; default: throw new IllegalStateException(String.format("Should never be here. Unrecognized metric scope %s", rawMetricType.metricScope())); } }
public ByteBuffer toBuffer(int headerPos) { byte[] topic = _topic.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.allocate(headerPos + 1 /* version */ + 1 /* raw metric type */ + Long.BYTES /* time */ + Integer.BYTES /* broker id */ + Integer.BYTES /* topic length */ + topic.length /* topic */ + Double.BYTES /* value */); buffer.position(headerPos); buffer.put(METRIC_VERSION); buffer.put(rawMetricType().id()); buffer.putLong(time()); buffer.putInt(brokerId()); buffer.putInt(topic.length); buffer.put(topic); buffer.putDouble(value()); return buffer; }
switch (rawMetricType) { case TOPIC_BYTES_IN: metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 1, BROKER_ID_0, TOPIC1, 20.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 2, BROKER_ID_1, TOPIC1, 500.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC2, 800.0 * BYTES_IN_KB)); break; case TOPIC_BYTES_OUT: metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, 80.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, 500.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, 1300.0 * BYTES_IN_KB)); break; case TOPIC_REPLICATION_BYTES_IN: metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC1, 20.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC1, 500.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC2, 800.0 * BYTES_IN_KB)); break; case TOPIC_REPLICATION_BYTES_OUT: metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, 20.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, 500.0 * BYTES_IN_KB)); metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, 800.0 * BYTES_IN_KB)); break; default: metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC1, i * BYTES_IN_MB)); metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC1, i * BYTES_IN_MB)); metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC2, i * BYTES_IN_MB)); metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC2, i * BYTES_IN_MB)); break;
/** * Send a CruiseControlMetric to the Kafka topic. * @param ccm the Cruise Control metric to send. */ public void sendCruiseControlMetric(CruiseControlMetric ccm) { // Use topic name as key if existing so that the same sampler will be able to collect all the information // of a topic. String key = ccm.metricClassId() == CruiseControlMetric.MetricClassId.TOPIC_METRIC ? ((TopicMetric) ccm).topic() : Integer.toString(ccm.brokerId()); ProducerRecord<String, CruiseControlMetric> producerRecord = new ProducerRecord<>(_cruiseControlMetricsTopic, null, ccm.time(), key, ccm); LOG.debug("Sending Cruise Control metric {}.", ccm); _producer.send(producerRecord, new Callback() { @Override public void onCompletion(RecordMetadata recordMetadata, Exception e) { if (e != null) { LOG.warn("Failed to send Cruise Control metric {}", ccm); _numMetricSendFailure++; } } }); }
public static CruiseControlMetric fromBytes(byte[] bytes) throws UnknownVersionException { ByteBuffer buffer = ByteBuffer.wrap(bytes); switch (CruiseControlMetric.MetricClassId.forId(buffer.get())) { case BROKER_METRIC: return BrokerMetric.fromBuffer(buffer); case TOPIC_METRIC: return TopicMetric.fromBuffer(buffer); case PARTITION_METRIC: return PartitionMetric.fromBuffer(buffer); default: // This could happen when a new type of metric is added but we are still running the old code. // simply ignore the metric by returning a null. return null; } }
@Override public String toString() { return String.format("[%s,%s,time=%d,brokerId=%d,topic=%s,value=%.3f]", MetricClassId.TOPIC_METRIC, rawMetricType(), time(), brokerId(), topic(), value()); } }
case BYTES_IN_PER_SEC: if (topic != null) { return new TopicMetric(RawMetricType.TOPIC_BYTES_IN, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_REPLICATION_BYTES_IN, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_REPLICATION_BYTES_OUT, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_FETCH_REQUEST_RATE, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_FETCH_REQUEST_RATE, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_PRODUCE_REQUEST_RATE, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_PRODUCE_REQUEST_RATE, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_MESSAGES_IN_PER_SEC, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_MESSAGES_IN_PER_SEC, now, brokerId, value);
public ByteBuffer toBuffer(int headerPos) { byte[] topic = _topic.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.allocate(headerPos + 1 /* version */ + 1 /* raw metric type */ + Long.BYTES /* time */ + Integer.BYTES /* broker id */ + Integer.BYTES /* topic length */ + topic.length /* topic */ + Double.BYTES /* value */); buffer.position(headerPos); buffer.put(METRIC_VERSION); buffer.put(rawMetricType().id()); buffer.putLong(time()); buffer.putInt(brokerId()); buffer.putInt(topic.length); buffer.put(topic); buffer.putDouble(value()); return buffer; }
static TopicMetric fromBuffer(ByteBuffer buffer) throws UnknownVersionException { byte version = buffer.get(); if (version > METRIC_VERSION) { throw new UnknownVersionException("Cannot deserialize the topic metrics for version " + version + ". " + "Current version is " + METRIC_VERSION); } RawMetricType rawMetricType = RawMetricType.forId(buffer.get()); long time = buffer.getLong(); int brokerId = buffer.getInt(); int topicLength = buffer.getInt(); String topic = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), topicLength, StandardCharsets.UTF_8); buffer.position(buffer.position() + topicLength); double value = buffer.getDouble(); return new TopicMetric(rawMetricType, time, brokerId, topic, value); }
case BYTES_IN_PER_SEC: if (topic != null) { return new TopicMetric(RawMetricType.TOPIC_BYTES_IN, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_REPLICATION_BYTES_IN, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_REPLICATION_BYTES_OUT, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_FETCH_REQUEST_RATE, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_FETCH_REQUEST_RATE, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_PRODUCE_REQUEST_RATE, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_PRODUCE_REQUEST_RATE, now, brokerId, value); return new TopicMetric(RawMetricType.TOPIC_MESSAGES_IN_PER_SEC, now, brokerId, topic, value); } else { return new BrokerMetric(RawMetricType.ALL_TOPIC_MESSAGES_IN_PER_SEC, now, brokerId, value);