private boolean isDuplicate(final E eventId) { long eventTime = context.timestamp(); WindowStoreIterator<Long> timeIterator = eventIdStore.fetch( eventId, eventTime - leftDurationMs, eventTime + rightDurationMs); boolean isDuplicate = timeIterator.hasNext(); timeIterator.close(); return isDuplicate; }
public KeyValue<K, V> transform(final K key, final V value) { E eventId = idExtractor.apply(key, value); if (eventId == null) { return KeyValue.pair(key, value); } else { KeyValue<K, V> output; if (isDuplicate(eventId)) { output = null; updateTimestampOfExistingEventToPreventExpiry(eventId, context.timestamp()); } else { output = KeyValue.pair(key, value); rememberNewEvent(eventId, context.timestamp()); } return output; } }
@Override public long timestamp() { return delegate.timestamp(); }
@Override public long time(final ProcessorContext context, final K key) { return context.timestamp(); }
@Override public void put(final K key, final V value) { put(key, value, context.timestamp()); }
@Override public void put(final K key, final V value) { put(key, value, context.timestamp()); }
@Override public void put(final Bytes key, final byte[] value) { put(key, value, context.timestamp()); }
void logChange(final K key, final V value) { if (collector != null) { // Twitter Added newEntries.put(key, new ValueAndTimestamp<>(value, context.timestamp())); } }
void logChange(final K key, final V value) { if (collector != null) { final Serializer<K> keySerializer = serialization.keySerializer(); final Serializer<V> valueSerializer = serialization.valueSerializer(); // Sending null headers to changelog topics (KIP-244) collector.send(this.topic, key, value, null, this.partition, context.timestamp(), keySerializer, valueSerializer); } } }
/** * {@inheritDoc} */ @Override public void process(K key, V value) { // If the key or value is null we don't need to proceed if (key == null || value == null) { return; } final NFA<K, V> nfa = loadNFA(this.stages, key); if (checkHighWaterMark()) { Event<K, V> event = new Event<>(key, value, context.timestamp(), context.topic(), context.partition(), context.offset()); List<Sequence<K, V>> sequences = nfa.matchPattern(event); Map<String, Long> latestOffsets = this.currentNFAState.getLatestOffsets(); latestOffsets.put(this.context.topic(), this.context.offset() + 1); this.currentNFAState = new NFAStates<>(nfa.getComputationStages(), nfa.getRuns(), latestOffsets); this.nfaStore.put(getRunned(key), this.currentNFAState); sequences.forEach(seq -> this.context.forward(key, seq)); } }