@Override public KeyValue<String, Long> transform(byte[] key, String value) { // For simplification (and unlike the traditional wordcount) we assume that the value is // a single word, i.e. we don't split the value by whitespace into potentially one or more // words. Optional<Long> count = Optional.ofNullable(stateStore.get(value)); Long incrementedCount = count.orElse(0L) + 1; stateStore.put(value, incrementedCount); return KeyValue.pair(value, incrementedCount); }
@Override public byte[] get(final Bytes key) { return inner.get(key); }
@Override public V get(final K key) { return this.inner.get(key); }
@Override public V get(K key) { return this.inner.get(key); }
@Override public KeyValue<String, OrderValidation> transform(final Product productId, final KeyValue<Order, Integer> orderAndStock) { //Process each order/inventory pair one at a time OrderValidation validated; Order order = orderAndStock.key; Integer warehouseStockCount = orderAndStock.value; //Look up locally 'reserved' stock from our state store Long reserved = reservedStocksStore.get(order.getProduct()); if (reserved == null) { reserved = 0L; } //If there is enough stock available (considering both warehouse inventory and reserved stock) validate the order if (warehouseStockCount - reserved - order.getQuantity() >= 0) { //reserve the stock by adding it to the 'reserved' store reservedStocksStore.put(order.getProduct(), reserved + order.getQuantity()); //validate the order validated = new OrderValidation(order.getId(), INVENTORY_CHECK, PASS); } else { //fail the order validated = new OrderValidation(order.getId(), INVENTORY_CHECK, FAIL); } return KeyValue.pair(validated.getOrderId(), validated); }
@Override public Optional<AggregationState> get(final VisibilityBindingSet bs) { requireNonNull(bs); final String key = makeCommaDelimitedValues(groupByVars, bs); return Optional.ofNullable(store.get(key)); }
@Override public void process(String key, String value) { Stream.of(value.toLowerCase().split(" ")).forEach((String word) -> { Optional<Integer> counts = Optional.ofNullable(kvStore.get(word)); int count = counts.map(wordcount -> wordcount + 1).orElse(1); kvStore.put(word, count); }); }
/** Processes the incoming message, calculating the exponential moving * average of the value of the message. * The value of the average is checkpointed to the state store with each * message. The message is forwarded to the sink "fast-sink". * * {@inheritDoc} */ @Override public void process(String key, Double value) { // If it's in the state store, use that value. Otherwise, use 0.0. // This is particularly important when restoring from a failover. Double oldValue = state.get(key); if(oldValue == null) { oldValue = 0.0; } // Now calculate the new moving average. double newAvg = alpha * value + (1 - alpha) * oldValue; // Update the state store. state.put(key, newAvg); } // Close process.
/** Processes the incoming message, calculating the exponential moving * average of the value of the message and saving the value in the * state store. * * {@inheritDoc} */ @Override public void process(String key, Message value) { Double oldValue = state.get(key); // Verify that the value is present. if(oldValue == null) { oldValue = 0.0; } // Now calculate the new moving average. double newAvg = alpha * value.getValue() + (1 - alpha) * oldValue; // Update the state store. state.put(key, newAvg); } // Close process.
/** * {@inheritDoc} */ @Override public NFAStates<K, V> find(final Runned key) { byte[] bytes = bytesStore.get(Bytes.wrap(serdes.rawKey(key))); return serdes.valueFrom(bytes); } }
/** * {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public <T> T find(Aggregated<K> aggregated) { byte[] bytes = bytesStore.get(Bytes.wrap(serdes.rawKey(aggregated))); return (T) serdes.valueFrom(bytes); }
private byte[] getInternal(final Bytes key) { LRUCacheEntry entry = null; if (cache != null) { entry = cache.get(cacheName, key); } if (entry == null) { final byte[] rawValue = underlying.get(key); if (rawValue == null) { return null; } // only update the cache if this call is on the streamThread // as we don't want other threads to trigger an eviction/flush if (Thread.currentThread().equals(streamThread)) { cache.put(cacheName, key, new LRUCacheEntry(rawValue)); } return rawValue; } else { return entry.value(); } }
@Override public V get(final K key) { try { if (getTime.shouldRecord()) { return measureLatency(() -> outerValue(inner.get(Bytes.wrap(serdes.rawKey(key)))), getTime); } else { return outerValue(inner.get(Bytes.wrap(serdes.rawKey(key)))); } } catch (final ProcessorStateException e) { final String message = String.format(e.getMessage(), key); throw new ProcessorStateException(message, e); } }
@Override public KeyValue<String, Review> transform(String courseId, Review review) { Long reviewId = review.getId(); Long now = System.currentTimeMillis(); if (reviewStore.get(reviewId) == null && !isReviewExpired(review, now, timeToKeepAReview)) { reviewStore.put(review.getId(), review); updateMinTimestamp(review); return KeyValue.pair(courseId, review); } else { return null; } }
public void process(String key, StockTransaction stockTransaction) { String currentSymbol = stockTransaction.getSymbol(); StockTransactionSummary transactionSummary = summaryStore.get(currentSymbol); if (transactionSummary == null) { transactionSummary = StockTransactionSummary.fromTransaction(stockTransaction); } else { transactionSummary.update(stockTransaction); } summaryStore.put(currentSymbol, transactionSummary); this.context.commit(); }
Matched currEventKey = Matched.from(currStage, currEvent); byte[] prevBytes = this.bytesStore.get(Bytes.wrap(serdes.rawKey(prevEventKey))); MatchedEvent sharedPrevEvent = serdes.valueFrom(prevBytes); byte[] currBytes = this.bytesStore.get(Bytes.wrap(serdes.rawKey(currEventKey))); MatchedEvent sharedCurrEvent = serdes.valueFrom(currBytes);
/** * {@inheritDoc} */ @Override public void branch(final Stage<K, V> stage, final Event<K, V> event, final DeweyVersion version) { Matched key = Matched.from(stage, event); MatchedEvent.Pointer pointer = new MatchedEvent.Pointer(version, key); while(pointer != null && (key = pointer.getKey()) != null) { byte[] bytes = this.bytesStore.get(Bytes.wrap(serdes.rawKey(key))); final MatchedEvent<K, V> val = serdes.valueFrom(bytes); val.incrementRefAndGet(); this.bytesStore.put(Bytes.wrap(serdes.rawKey(key)), serdes.rawValue(val)); pointer = val.getPointerByVersion(pointer.getVersion()); } }
private Sequence<K, V> peek(final Matched matched, DeweyVersion version, boolean remove) { MatchedEvent.Pointer pointer = new MatchedEvent.Pointer(version, matched); Sequence.Builder<K, V> builder = new Sequence.Builder<>(); while (pointer != null && pointer.getKey() != null) { final Matched key = pointer.getKey(); byte[] bytes = this.bytesStore.get(Bytes.wrap(serdes.rawKey(key))); final MatchedEvent<K, V> stateValue = serdes.valueFrom(bytes); long refsLeft = stateValue.decrementRefAndGet(); if (remove && refsLeft == 0 && stateValue.getPredecessors().size() <= 1) { this.bytesStore.delete(Bytes.wrap(serdes.rawKey(key))); } builder.add(key.getStageName(), newEvent(key, stateValue)); pointer = stateValue.getPointerByVersion(pointer.getVersion()); if (remove && pointer != null && refsLeft == 0) { stateValue.removePredecessor(pointer); this.bytesStore.put(Bytes.wrap(serdes.rawKey(key)), serdes.rawValue(stateValue)); } } return builder.build(true); }
private void putAndMaybeForward(final ThreadCache.DirtyEntry entry, final InternalProcessorContext context) { final ProcessorRecordContext current = context.recordContext(); try { context.setRecordContext(entry.entry().context()); if (flushListener != null) { V oldValue = null; if (sendOldValues) { final byte[] oldBytesValue = underlying.get(entry.key()); oldValue = oldBytesValue == null ? null : serdes.valueFrom(oldBytesValue); } // we rely on underlying store to handle null new value bytes as deletes underlying.put(entry.key(), entry.newValue()); flushListener.apply(serdes.keyFrom(entry.key().get()), serdes.valueFrom(entry.newValue()), oldValue); } else { underlying.put(entry.key(), entry.newValue()); } } finally { context.setRecordContext(current); } }