@Override public KeyValue<String, Long> transform(byte[] key, String value) { // For simplification (and unlike the traditional wordcount) we assume that the value is // a single word, i.e. we don't split the value by whitespace into potentially one or more // words. Optional<Long> count = Optional.ofNullable(stateStore.get(value)); Long incrementedCount = count.orElse(0L) + 1; stateStore.put(value, incrementedCount); return KeyValue.pair(value, incrementedCount); }
@Override public KeyValueIterator<K, V> all() { return this.inner.all(); } }
/** {@inheritDoc} */ @Override public void close() { state.close(); } // Close close.
@Override public void put(K key, V value) { this.inner.put(key, value); changeLogger.logChange(key, value); }
@Override public V get(final K key) { return this.inner.get(key); }
private Sequence<K, V> peek(final Matched matched, DeweyVersion version, boolean remove) { MatchedEvent.Pointer pointer = new MatchedEvent.Pointer(version, matched); Sequence.Builder<K, V> builder = new Sequence.Builder<>(); while (pointer != null && pointer.getKey() != null) { final Matched key = pointer.getKey(); byte[] bytes = this.bytesStore.get(Bytes.wrap(serdes.rawKey(key))); final MatchedEvent<K, V> stateValue = serdes.valueFrom(bytes); long refsLeft = stateValue.decrementRefAndGet(); if (remove && refsLeft == 0 && stateValue.getPredecessors().size() <= 1) { this.bytesStore.delete(Bytes.wrap(serdes.rawKey(key))); } builder.add(key.getStageName(), newEvent(key, stateValue)); pointer = stateValue.getPointerByVersion(pointer.getVersion()); if (remove && pointer != null && refsLeft == 0) { stateValue.removePredecessor(pointer); this.bytesStore.put(Bytes.wrap(serdes.rawKey(key)), serdes.rawValue(stateValue)); } } return builder.build(true); }
@Override public KeyValue<String, Review> punctuate(long currentTime) { if (minTimestampInStore + timeToKeepAReview < currentTime && reviewStore.approximateNumEntries() > 0) { log.info("let's expire data!"); // invalidate the min timestamp in store as we're going to re-compute it minTimestampInStore = System.currentTimeMillis(); KeyValueIterator<Long, Review> it = reviewStore.all(); List<Long> keysToRemove = new ArrayList<>(); while (it.hasNext()) { KeyValue<Long, Review> next = it.next(); Review review = next.value; Long courseId = review.getCourse().getId(); if (isReviewExpired(review, currentTime, timeToKeepAReview)) { Long reviewId = next.key; keysToRemove.add(reviewId); // we push an opposite review event to remove data from the average Review reverseReview = reverseReview(review); this.context.forward(courseId, reverseReview); } else { // update the min timestamp in store as we know the data is staying updateMinTimestamp(review); } } for (Long key : keysToRemove) { reviewStore.delete(key); } } // this is okay because we called this.context.forward() multiple times before return null; }
@Override public boolean hasNext() { boolean hasNext = false; while ((currentIterator == null || !(hasNext = hasNextConditionHasNext()) || !currentSegment.isOpen()) && segments.hasNext()) { close(); currentSegment = segments.next(); try { if (from == null || to == null) { currentIterator = currentSegment.all(); } else { currentIterator = currentSegment.range(from, to); } } catch (final InvalidStateStoreException e) { // segment may have been closed so we ignore it. } } return currentIterator != null && hasNext; }
@Override public void init(final ProcessorContext context, final StateStore root) { inner.init(context, root); final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()); this.changeLogger = new StoreChangeLogger<>(inner.name(), context, new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray())); // if the inner store is an LRU cache, add the eviction listener to log removed record if (inner instanceof MemoryLRUCache) { ((MemoryLRUCache<Bytes, byte[]>) inner).whenEldestRemoved(new MemoryLRUCache.EldestEntryRemovalListener<Bytes, byte[]>() { @Override public void apply(final Bytes key, final byte[] value) { // pass null to indicate removal changeLogger.logChange(key, null); } }); } }
@Override public V delete(final K key) { final V value = this.inner.delete(key); removed(key); return value; }
@Override public long approximateNumEntries() { return inner.approximateNumEntries(); }
@Override public void init(final ProcessorContext context, final StateStore root) { initInternal(context); underlying.init(context, root); // save the stream thread as we only ever want to trigger a flush // when the stream thread is the current thread. streamThread = Thread.currentThread(); }
@SuppressWarnings("unchecked") private void initInternal(final ProcessorContext context) { this.context = (InternalProcessorContext) context; this.serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), underlying.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.cache = this.context.getCache(); this.cacheName = ThreadCache.nameSpaceFromTaskIdAndStore(context.taskId().toString(), underlying.name()); cache.addDirtyEntryFlushListener(cacheName, new ThreadCache.DirtyEntryFlushListener() { @Override public void apply(final List<ThreadCache.DirtyEntry> entries) { for (final ThreadCache.DirtyEntry entry : entries) { putAndMaybeForward(entry, (InternalProcessorContext) context); } } }); }
@Override public void put(final Bytes key, final byte[] value) { inner.put(key, value); changeLogger.logChange(key, value); }
@Override public byte[] get(final Bytes key) { return inner.get(key); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { inner.init(context, root); // construct the serde final StateSerdes<K, V> serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.changeLogger = new StoreChangeLogger<>(inner.name(), context, serdes); // if the inner store is an LRU cache, add the eviction listener to log removed record if (inner instanceof MemoryLRUCache) { ((MemoryLRUCache<K, V>) inner).whenEldestRemoved(new MemoryNavigableLRUCache.EldestEntryRemovalListener<K, V>() { @Override public void apply(final K key, final V value) { removed(key); } }); } }
@Override public byte[] delete(final Bytes key) { final byte[] oldValue = inner.delete(key); changeLogger.logChange(key, null); return oldValue; }
@Override public long approximateNumEntries() { return inner.approximateNumEntries(); }