private void rememberNewEvent(final E eventId, long timestamp) { eventIdStore.put(eventId, timestamp, timestamp); }
private List<KeyValue<Windowed<String>, Long>> fetchSessionsFromLocalStore(final String userId, final ReadOnlySessionStore<String, Long> playEventsPerSession) { final List<KeyValue<Windowed<String>, Long>> results = new ArrayList<>(); try (final KeyValueIterator<Windowed<String>, Long> iterator = playEventsPerSession.fetch(userId)) { iterator.forEachRemaining(results::add); } return results; }
private ReadOnlyKeyValueStore<String, Order> ordersStore() { return streams.store(ORDERS_STORE_NAME, QueryableStoreTypes.keyValueStore()); }
/** * Performs a range query on a KeyValue Store and converts the results into a List of * {@link KeyValueBean} * @param storeName The store to query * @param rangeFunction The range query to run, i.e., all, from(start, end) * @return List of {@link KeyValueBean} */ private List<KeyValueBean> rangeForKeyValueStore(final String storeName, final Function<ReadOnlyKeyValueStore<String, Long>, KeyValueIterator<String, Long>> rangeFunction) { // Get the KeyValue Store final ReadOnlyKeyValueStore<String, Long> store = streams.store(storeName, QueryableStoreTypes.keyValueStore()); final List<KeyValueBean> results = new ArrayList<>(); // Apply the function, i.e., query the store final KeyValueIterator<String, Long> range = rangeFunction.apply(store); // Convert the results while (range.hasNext()) { final KeyValue<String, Long> next = range.next(); results.add(new KeyValueBean(next.key, next.value)); } return results; }
@GET() @Path("/song/{id}") @Produces(MediaType.APPLICATION_JSON) public SongBean song(@PathParam("id") Long songId) { final ReadOnlyKeyValueStore<Long, Song> songStore = streams.store(KafkaMusicExample.ALL_SONGS, QueryableStoreTypes.<Long, Song>keyValueStore()); final Song song = songStore.get(songId); if (song == null) { throw new NotFoundException(String.format("Song with id [%d] was not found", songId)); } return new SongBean(song.getArtist(), song.getAlbum(), song.getName()); }
@Override public KeyValue<String, Long> transform(byte[] key, String value) { // For simplification (and unlike the traditional wordcount) we assume that the value is // a single word, i.e. we don't split the value by whitespace into potentially one or more // words. Optional<Long> count = Optional.ofNullable(stateStore.get(value)); Long incrementedCount = count.orElse(0L) + 1; stateStore.put(value, incrementedCount); return KeyValue.pair(value, incrementedCount); }
private void verifyAllProductsInStore(final List<Product> products, final ReadOnlyKeyValueStore<Long, Product> store) { for (long id = 0; id < products.size(); id++) { assertThat(store.get(id), equalTo(products.get((int)id))); } }
QueryableStoreTypes.<String, Long>windowStore()); if (store == null) { throw new NotFoundException(); final WindowStoreIterator<Long> results = store.fetch(key, from, to); while (results.hasNext()) { final KeyValue<Long, Long> next = results.next();
/** * Asserts that the oldest available window in the window store contains the expected content. * * @param store the store to be validated * @param expected the expected contents of the store * @param <K> the store's key type * @param <V> the store's value type */ public static <K, V> void assertThatOldestWindowContains(ReadOnlyWindowStore<K, V> store, Map<K, V> expected) throws InterruptedException { final long fromBeginningOfTimeMs = 0; final long toNowInProcessingTimeMs = System.currentTimeMillis(); TestUtils.waitForCondition(() -> expected.keySet().stream().allMatch(k -> { try (WindowStoreIterator<V> iterator = store.fetch(k, fromBeginningOfTimeMs, toNowInProcessingTimeMs)) { if (iterator.hasNext()) { return expected.get(k).equals(iterator.next().value); } return false; } }), 30000, "Expected values not found in WindowStore"); }
private boolean isDuplicate(final E eventId) { long eventTime = context.timestamp(); WindowStoreIterator<Long> timeIterator = eventIdStore.fetch( eventId, eventTime - leftDurationMs, eventTime + rightDurationMs); boolean isDuplicate = timeIterator.hasNext(); timeIterator.close(); return isDuplicate; }
private List<HostStoreInfo> mapInstancesToHostStoreInfo( final Collection<StreamsMetadata> metadatas) { return metadatas.stream().map(metadata -> new HostStoreInfo(metadata.host(), metadata.port(), metadata.stateStoreNames())) .collect(Collectors.toList()); }
static WordCountInteractiveQueriesRestService startRestProxy(final KafkaStreams streams, final int port, final String host) throws Exception { final HostInfo hostInfo = new HostInfo(host, port); final WordCountInteractiveQueriesRestService wordCountInteractiveQueriesRestService = new WordCountInteractiveQueriesRestService(streams, hostInfo); wordCountInteractiveQueriesRestService.start(port); return wordCountInteractiveQueriesRestService; }
@Override public KeyValue<String, OrderValidation> transform(final Product productId, final KeyValue<Order, Integer> orderAndStock) { //Process each order/inventory pair one at a time OrderValidation validated; Order order = orderAndStock.key; Integer warehouseStockCount = orderAndStock.value; //Look up locally 'reserved' stock from our state store Long reserved = reservedStocksStore.get(order.getProduct()); if (reserved == null) { reserved = 0L; } //If there is enough stock available (considering both warehouse inventory and reserved stock) validate the order if (warehouseStockCount - reserved - order.getQuantity() >= 0) { //reserve the stock by adding it to the 'reserved' store reservedStocksStore.put(order.getProduct(), reserved + order.getQuantity()); //validate the order validated = new OrderValidation(order.getId(), INVENTORY_CHECK, PASS); } else { //fail the order validated = new OrderValidation(order.getId(), INVENTORY_CHECK, FAIL); } return KeyValue.pair(validated.getOrderId(), validated); }
private void verifyAllCustomersInStore(final List<Customer> customers, final ReadOnlyKeyValueStore<Long, Customer> store) { for (long id = 0; id < customers.size(); id++) { assertThat(store.get(id), equalTo(customers.get((int)id))); } }
/** * Find the metadata for the instance of this Kafka Streams Application that has the given * store and would have the given key if it exists. * @param store Store to find * @param key The key to find * @return {@link HostStoreInfo} */ public <K> HostStoreInfo streamsMetadataForStoreAndKey(final String store, final K key, final Serializer<K> serializer) { // Get metadata for the instances of this Kafka Streams application hosting the store and // potentially the value for key final StreamsMetadata metadata = streams.metadataForKey(store, key, serializer); if (metadata == null) { throw new NotFoundException(); } return new HostStoreInfo(metadata.host(), metadata.port(), metadata.stateStoreNames()); }
private void updateTimestampOfExistingEventToPreventExpiry(final E eventId, long newTimestamp) { eventIdStore.put(eventId, newTimestamp, newTimestamp); }
/** * Asserts that the key-value store contains exactly the expected content and nothing more. * * @param store the store to be validated * @param expected the expected contents of the store * @param <K> the store's key type * @param <V> the store's value type */ public static <K, V> void assertThatKeyValueStoreContains(ReadOnlyKeyValueStore<K, V> store, Map<K, V> expected) throws InterruptedException { TestUtils.waitForCondition(() -> expected.keySet() .stream() .allMatch(k -> expected.get(k).equals(store.get(k))), 30000, "Expected values not found in KV store"); }