protected void initAutoAdvanceTime(BatchSource source) { String key = getKey(source); if (!m_lastAdvanceTimes.containsKey(key)) m_lastAdvanceTimes.put(key, System.currentTimeMillis()); }
protected void cacheBatch(BatchSource source, Collection<JetstreamEvent> events) { String key = getKey(source); List<JetstreamEvent> cache = cachedBatches.get(key); if (cache == null) { cache = new ArrayList<JetstreamEvent>(); cachedBatches.put(key, cache); } cache.addAll(events); }
@Override public void onBatchProcessed(BatchSource source) { if (m_config.isNoDuplication()) { String key = getKey(source); List<JetstreamEvent> cached = cachedBatches.get(key); if (cached != null) { simplySendBatch(source, cached); cached.clear(); } } }
private void handleReadRate(long start, BatchSource source, int eventCount) { if (m_config.getMaxReadRate() > 0) { long end = System.currentTimeMillis(); int tasks; String key = getKey(source); if (waitForLastBatch.containsKey(key)) { tasks = waitForLastBatch.size(); } else { tasks = waitForLastBatch.size() + 1; } long durInRate = (1000 * eventCount * tasks) / m_config.getMaxReadRate(); long waitMs = durInRate - (end - start); waitForLastBatch.put(key, waitMs); } }
@Override public BatchResponse onNextBatch(BatchSource source, Collection<JetstreamEvent> events) throws EventException { initAutoAdvanceTime(source); incrementEventRecievedCounter(events.size()); if (m_config.isNoDuplication()) { cacheBatch(source, events); } else { BatchResponse ret = sendBatch(source, events); if (ret != null) return ret; } BatchResponse ret = autoAdvanceResponse(source); // check if the read rate exceeds the max, if exceeds, make the next // batch wait for a while String key = getKey(source); Long waitMs = waitForLastBatch.get(key); if (waitMs != null && waitMs > 0) { ret.setWaitTimeInMs(waitMs); } return ret; }