private TimeBucketMetadata getTimeBucket(double timestamp) { timeBucketsRWLock.readLock().lock(); try { double bucket = getBucketMapping(timestamp); if (!timeBucketsMap.containsKey(bucket)) { if (null == timeBucketsMap.putIfAbsent(bucket, new TimeBucketMetadata(bucket, bucket + timeBucketIntervalSec))) { timeBucketCount.incrementAndGet(); } } return timeBucketsMap.get(bucket); } finally { timeBucketsRWLock.readLock().unlock(); } }
public void track(double timestamp, int msgCount) { long currentTimeMillis = 0; timeBucketsRWLock.readLock().lock(); try { TimeBucketMetadata timeBucket = getTimeBucket(timestamp); timeBucket.msgCount.addAndGet(msgCount); currentTimeMillis = System.currentTimeMillis(); timeBucket.lastMessageTimestampSeenInSec = timestamp; double latency = currentTimeMillis - (timestamp * 1000); for (int i = 0; i < msgCount; i++) { timeBucket.latencyStats.addValue(latency); } } finally { timeBucketsRWLock.readLock().unlock(); } if (timeBucketCount.get() >= reportFreqBucketCount || currentTimeMillis > nextReportTime.get()) { report(); } }
public void track(double timestamp) { track(timestamp, 1); }
@Test public void testLastSeen() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(testTimestampInSec, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).lastMessageTimestampSeenInSec, 0.001); }
@Test public void testTrackReport() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(1, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); messageTracker.report(); buckets = messageTracker.getTimeBucketsMap(); assertEquals(0, buckets.size()); }
@Test public void testTrackMultipleBuckets() throws Exception { int totalMessages = 10; for (int i = 0; i < totalMessages; i++) { messageTracker.track(testTimestampInSec + 60000 * i); } // 10 time-buckets seen so far. ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(totalMessages, buckets.size()); messageTracker.report(); // All were cleared buckets = messageTracker.getTimeBucketsMap(); assertEquals(0, buckets.size()); }
public void shutdown() { report(); }
public MessageTracker(String topicName, AuditReporter auditReporter, int timeBucketIntervalSec, int reportFreqBucketCount, int reportFreqIntervalSec) { this.auditReporter = auditReporter; this.topicName = topicName; timeBucketsMap = new ConcurrentHashMap<>(); timeBucketsRWLock = new ReentrantReadWriteLock(); this.timeBucketIntervalSec = timeBucketIntervalSec; this.reportFreqIntervalSec = reportFreqIntervalSec; this.reportFreqBucketCount = reportFreqBucketCount; timeBucketCount = new AtomicInteger(0); nextReportTime = new AtomicLong(setNextReportingTime()); reportingInProgress.set(false); logger.debug("Started auditing for topic {}", this.topicName); }
@Before public void setUp() throws Exception { messageTracker = new MessageTracker("test-topic", new AuditReporter() { @Override public AuditMessage buildAuditMessage(String topicName, TimeBucketMetadata timeBucket) { return null; } @Override public void reportAuditMessage(AuditMessage message) throws IOException {} @Override public void shutdown() {} }, bucketIntervalInSec, 123, 60); }
@Test public void testTrackOnce() throws Exception { messageTracker.track(testTimestampInSec); ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(1, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); }
public boolean report() { if (reportingInProgress.getAndSet(true)) { logger.info("Reporting already in progress for {}", topicName); return false; } Map<Double, TimeBucketMetadata> tempTimeCountsMap; timeBucketsRWLock.writeLock().lock(); try { tempTimeCountsMap = timeBucketsMap; timeBucketCount.set(0); timeBucketsMap = new ConcurrentHashMap<>(); } finally { timeBucketsRWLock.writeLock().unlock(); } try { for (TimeBucketMetadata bucket : tempTimeCountsMap.values()) { AuditMessage m = auditReporter.buildAuditMessage(topicName, bucket); auditReporter.reportAuditMessage(m); } } catch (IOException ioe) { logger.error("IOException when trying to send audit message: {}", ioe.toString()); } nextReportTime.set(setNextReportingTime()); reportingInProgress.set(false); return true; }
@Test public void testTrackMultiple() throws Exception { long totalMessages = 10; for (int i = 1; i <= totalMessages; i++) { messageTracker.track(testTimestampInSec + i); } ConcurrentMap<Double, TimeBucketMetadata> buckets = messageTracker.getTimeBucketsMap(); assertEquals(totalMessages, buckets.get(messageTracker.getBucketMapping(testTimestampInSec)).msgCount.get()); }