public void shutdown() { report(); }
public AuditMessage buildAuditMessage(String topicName, TimeBucketMetadata timeBucket) { return new AuditMessage(topicName, timeBucket, hostMetadata); }
public void track(double timestamp) { track(timestamp, 1); }
@Test public void testLastSeen() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(testTimestampInSec, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).lastMessageTimestampSeenInSec, 0.001); }
@Test public void testTrackReport() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(1, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); messageTracker.report(); buckets = messageTracker.getTimeBucketsMap(); assertEquals(0, buckets.size()); }
@Test public void testTrackMultipleBuckets() throws Exception { int totalMessages = 10; for (int i = 0; i < totalMessages; i++) { messageTracker.track(testTimestampInSec + 60000 * i); } // 10 time-buckets seen so far. ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(totalMessages, buckets.size()); messageTracker.report(); // All were cleared buckets = messageTracker.getTimeBucketsMap(); assertEquals(0, buckets.size()); }
public void reportAuditMessage(final AuditMessage message) throws IOException { auditMsg.put(AuditMsgField.TOPICNAME.getName(), message.topicName); auditMsg.put(AuditMsgField.TIME_BUCKET_START.getName(), message.timeBucketMetadata.timeBucketStartInSec); auditMsg.put(AuditMsgField.TIME_BUCKET_END.getName(), message.timeBucketMetadata.timeBucketEndInSec); auditMsg.put(AuditMsgField.METRICS_COUNT.getName(), message.timeBucketMetadata.msgCount.get()); auditMsg.put(AuditMsgField.METRICS_MEAN_LATENCY.getName(), message.timeBucketMetadata.latencyStats.getMean()); auditMsg .put(AuditMsgField.METRICS_P95_LATENCY.getName(), message.timeBucketMetadata.latencyStats.getPercentile(95)); auditMsg .put(AuditMsgField.METRICS_P99_LATENCY.getName(), message.timeBucketMetadata.latencyStats.getPercentile(99)); auditMsg.put(AuditMsgField.METRICS_MAX_LATENCY.getName(), message.timeBucketMetadata.latencyStats.getMax()); auditMsg.put(AuditMsgField.TIER.getName(), message.hostMetadata.getTier()); auditMsg.put(AuditMsgField.HOSTNAME.getName(), message.hostMetadata.getHost()); auditMsg.put(AuditMsgField.DATACENTER.getName(), message.hostMetadata.getDatacenter()); auditMsg.put(AuditMsgField.UUID.getName(), UUID.randomUUID().toString());
private TimeBucketMetadata getTimeBucket(double timestamp) { timeBucketsRWLock.readLock().lock(); try { double bucket = getBucketMapping(timestamp); if (!timeBucketsMap.containsKey(bucket)) { if (null == timeBucketsMap.putIfAbsent(bucket, new TimeBucketMetadata(bucket, bucket + timeBucketIntervalSec))) { timeBucketCount.incrementAndGet(); } } return timeBucketsMap.get(bucket); } finally { timeBucketsRWLock.readLock().unlock(); } }
public boolean report() { if (reportingInProgress.getAndSet(true)) { logger.info("Reporting already in progress for {}", topicName); return false; } Map<Double, TimeBucketMetadata> tempTimeCountsMap; timeBucketsRWLock.writeLock().lock(); try { tempTimeCountsMap = timeBucketsMap; timeBucketCount.set(0); timeBucketsMap = new ConcurrentHashMap<>(); } finally { timeBucketsRWLock.writeLock().unlock(); } try { for (TimeBucketMetadata bucket : tempTimeCountsMap.values()) { AuditMessage m = auditReporter.buildAuditMessage(topicName, bucket); auditReporter.reportAuditMessage(m); } } catch (IOException ioe) { logger.error("IOException when trying to send audit message: {}", ioe.toString()); } nextReportTime.set(setNextReportingTime()); reportingInProgress.set(false); return true; }
public void track(double timestamp, int msgCount) { long currentTimeMillis = 0; timeBucketsRWLock.readLock().lock(); try { TimeBucketMetadata timeBucket = getTimeBucket(timestamp); timeBucket.msgCount.addAndGet(msgCount); currentTimeMillis = System.currentTimeMillis(); timeBucket.lastMessageTimestampSeenInSec = timestamp; double latency = currentTimeMillis - (timestamp * 1000); for (int i = 0; i < msgCount; i++) { timeBucket.latencyStats.addValue(latency); } } finally { timeBucketsRWLock.readLock().unlock(); } if (timeBucketCount.get() >= reportFreqBucketCount || currentTimeMillis > nextReportTime.get()) { report(); } }
public KafkaAuditReporter(String topicForAuditMsg, String auditDC, String auditTier, String brokerList, String requiredAcks, MetricRegistry metricRegistry) throws IOException { this.topicForAuditMsg = topicForAuditMsg; this.hostMetadata = new HostMetadata(auditDC, auditTier); this.producer = getKafkaProducer(brokerList, requiredAcks, hostMetadata); if (metricRegistry == null) { logger.info("Using temporary metrics registry"); metricRegistry = new MetricRegistry(); } this.messageReportRate = metricRegistry.meter("kafkaAuditReporter.messageReportRate"); this.bucketReportRate = metricRegistry.meter("kafkaAuditReporter.bucketReportRate"); }
private Producer<String, byte[]> getKafkaProducer(String brokerList, String requiredAcks, HostMetadata host) { Properties properties = new Properties(); // Old props list properties.put("metadata.broker.list", brokerList); properties.put("serializer.class", "kafka.serializer.DefaultEncoder"); properties.put("key.serializer.class", "kafka.serializer.StringEncoder"); properties.put("compression.type", "gzip"); properties.put("request.required.acks", requiredAcks); // New props needed properties.put("bootstrap.servers", brokerList); properties.put("client.id", host.getTier() + "-" + host.getHost()); return new KafkaProducer<>(properties, new StringSerializer(), new ByteArraySerializer()); }
public MessageTracker(String topicName, AuditReporter auditReporter, int timeBucketIntervalSec, int reportFreqBucketCount, int reportFreqIntervalSec) { this.auditReporter = auditReporter; this.topicName = topicName; timeBucketsMap = new ConcurrentHashMap<>(); timeBucketsRWLock = new ReentrantReadWriteLock(); this.timeBucketIntervalSec = timeBucketIntervalSec; this.reportFreqIntervalSec = reportFreqIntervalSec; this.reportFreqBucketCount = reportFreqBucketCount; timeBucketCount = new AtomicInteger(0); nextReportTime = new AtomicLong(setNextReportingTime()); reportingInProgress.set(false); logger.debug("Started auditing for topic {}", this.topicName); }
@Before public void setUp() throws Exception { messageTracker = new MessageTracker("test-topic", new AuditReporter() { @Override public AuditMessage buildAuditMessage(String topicName, TimeBucketMetadata timeBucket) { return null; } @Override public void reportAuditMessage(AuditMessage message) throws IOException {} @Override public void shutdown() {} }, bucketIntervalInSec, 123, 60); }
@Test public void testTrackOnce() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(1, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); }
@Test public void testTrackMultiple() throws Exception { long totalMessages = 10; for (int i = 1; i <= totalMessages; i++) { messageTracker.track(testTimestampInSec + i); } ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(totalMessages, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); }