@Override public BatchResponse onNextBatch(BatchSource source, Collection<JetstreamEvent> events) throws EventException { // check if it's time to send the batch, check the delay time of the // lastEvent, if it's no time to send the batch, tell the IKC to wait // and resend this batch long start = System.currentTimeMillis(); if (getReplayConfig().getTimestampKey() != null && getReplayConfig().getDelayInMs() > 0) { JetstreamEvent[] eventArr = events .toArray(new JetstreamEvent[events.size()]); JetstreamEvent lastEvent = eventArr[eventArr.length - 1]; Long ts = (Long) lastEvent.get(getReplayConfig().getTimestampKey()); if (ts != null) { long v = ts + getReplayConfig().getDelayInMs() - start; if (v > 0) { incrementEventRecievedCounter(events.size()); incrementEventDelayedCounter(events.size()); return BatchResponse.getNextBatch() .setOffset(source.getHeadOffset()) .setWaitTimeInMs(v); } } } return super.onNextBatch(source, events); }
@Override public BatchResponse onStreamTermination(BatchSource source) throws EventException { return BatchResponse.advanceAndGetNextBatch(); }
@Override public BatchResponse onException(BatchSource source, Exception ex) { return BatchResponse.getNextBatch().setOffset(source.getHeadOffset()); }
if (res.getWaitTimeInMs() > 0) p.setNextFetchInMs(System.currentTimeMillis() + res.getWaitTimeInMs()); int resBatchSize = res.getBatchSizeBytes(); if (resBatchSize >= 0 && resBatchSize <= m_config.getMaxBatchSizeBytes()) p.setNextBatchSizeBytes(res.getBatchSizeBytes()); long offset = res.getOffset(); if (res.getRequest() == null) return; switch (res.getRequest()) { case GetNextBatch: if (offset >= 0) {
protected BatchResponse autoAdvanceResponse(BatchSource source) { if (isAutoAdvanceEveryBatch()) { return BatchResponse.advanceAndGetNextBatch(); } else { String key = new StringBuilder().append(source.getTopic()) .append("-").append(source.getPartition()).toString(); Long lastTs = m_lastAdvanceTimes.get(key); long curTime = System.currentTimeMillis(); boolean advance = false; if (lastTs != null && (curTime - lastTs.longValue()) >= m_config .getAutoAdvanceInterval()) { advance = true; m_lastAdvanceTimes.put(key, curTime); } if (advance) { return BatchResponse.advanceAndGetNextBatch(); } else { return BatchResponse.getNextBatch(); } } }
@Override public BatchResponse onNextBatch(BatchSource source, Collection<JetstreamEvent> events) throws EventException { initAutoAdvanceTime(source); incrementEventRecievedCounter(events.size()); if (m_config.isNoDuplication()) { cacheBatch(source, events); } else { BatchResponse ret = sendBatch(source, events); if (ret != null) return ret; } BatchResponse ret = autoAdvanceResponse(source); // check if the read rate exceeds the max, if exceeds, make the next // batch wait for a while String key = getKey(source); Long waitMs = waitForLastBatch.get(key); if (waitMs != null && waitMs > 0) { ret.setWaitTimeInMs(waitMs); } return ret; }
private void handleOffsetOutOfRange(PartitionReader p, Exception e) { try { BatchResponse res = onException(p, e); // won't handle res offset, always reset offset if (res != null) res.setRequest(null); handleResponse(p, 0, res); p.resetOffset(); } catch (Throwable ex) { LOGGER.error( "Error occurs when handle OffsetOutOfRangeException for " + p.getId(), ex); } }
} catch (Throwable e) { incrementEventDroppedCounter(events.size()); return BatchResponse.getNextBatch().setOffset( source.getHeadOffset()); return BatchResponse.getNextBatch().setOffset( startOffset + index);