/** * Create a new instance of {@link StateSerdes} for the given state name and key-/value-type classes. * * @param topic the topic name * @param keyClass the class of the key type * @param valueClass the class of the value type * @param <K> the key type * @param <V> the value type * @return a new instance of {@link StateSerdes} */ public static <K, V> StateSerdes<K, V> withBuiltinTypes( final String topic, final Class<K> keyClass, final Class<V> valueClass) { return new StateSerdes<>(topic, Serdes.serdeFrom(keyClass), Serdes.serdeFrom(valueClass)); }
@Override public void init(final ProcessorContext context, final StateStore root) { this.context = context; bytesStore.init(context, root); final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), bytesStore.name()); changeLogger = new StoreChangeLogger<>( name(), context, new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray())); }
@Override public void init(final ProcessorContext context, final StateStore root) { bytesStore.init(context, root); final String topic = ProcessorStateManager.storeChangelogTopic( context.applicationId(), bytesStore.name()); changeLogger = new StoreChangeLogger<>( name(), context, new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray())); }
@Override public void init(final ProcessorContext context, final StateStore root) { inner.init(context, root); final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()); this.changeLogger = new StoreChangeLogger<>(inner.name(), context, new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray())); // if the inner store is an LRU cache, add the eviction listener to log removed record if (inner instanceof MemoryLRUCache) { ((MemoryLRUCache<Bytes, byte[]>) inner).whenEldestRemoved(new MemoryLRUCache.EldestEntryRemovalListener<Bytes, byte[]>() { @Override public void apply(final Bytes key, final byte[] value) { // pass null to indicate removal changeLogger.logChange(key, null); } }); } }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { this.context = context; // construct the serde serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), bytesStore.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); bytesStore.init(context, root); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { final String storeName = bytesStore.name(); topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); serdes = new StateSerdes<>( topic, keySerde == null ? (Serde<K>) context.keySerde() : keySerde, aggSerde == null ? (Serde<AGG>) context.valueSerde() : aggSerde); bytesStore.init(context, root); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { // construct the serde this.serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), name), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); // register the store context.register(root, new StateRestoreCallback() { @Override public void restore(final byte[] key, final byte[] value) { restoring = true; // check value for null, to avoid deserialization error. if (value == null) { delete(serdes.keyFrom(key)); } else { put(serdes.keyFrom(key), serdes.valueFrom(value)); } restoring = false; } }); }
@SuppressWarnings("unchecked") private void initInternal(final ProcessorContext context) { this.context = (InternalProcessorContext) context; final String topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), underlying.name()); serdes = new StateSerdes<>(topic, keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); bytesSerdes = new StateSerdes<>(topic, Serdes.Bytes(), Serdes.ByteArray()); name = context.taskId() + "-" + underlying.name(); cache = this.context.getCache(); cache.addDirtyEntryFlushListener(name, new ThreadCache.DirtyEntryFlushListener() { @Override public void apply(final List<ThreadCache.DirtyEntry> entries) { for (final ThreadCache.DirtyEntry entry : entries) { final byte[] binaryWindowKey = cacheFunction.key(entry.key()).get(); final long timestamp = WindowKeySchema.extractStoreTimestamp(binaryWindowKey); final Windowed<K> windowedKey = WindowKeySchema.fromStoreKey(binaryWindowKey, windowSize, serdes); final Bytes key = Bytes.wrap(WindowKeySchema.extractStoreKeyBytes(binaryWindowKey)); maybeForward(entry, key, windowedKey, (InternalProcessorContext) context); underlying.put(key, entry.newValue(), timestamp); } } }); }
public void init(final ProcessorContext context, final StateStore root) { final String storeName = bytesStore.name(); topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); bytesStore.init(context, root); serdes = new StateSerdes<>(topic, new AggregateKeySerde<>(), new KryoSerDe<>()); }
@SuppressWarnings("unchecked") private void initInternal(final InternalProcessorContext context) { this.context = context; keySchema.init(topic); serdes = new StateSerdes<>( topic, keySerde == null ? (Serde<K>) context.keySerde() : keySerde, aggSerde == null ? (Serde<AGG>) context.valueSerde() : aggSerde); cacheName = context.taskId() + "-" + bytesStore.name(); cache = context.getCache(); cache.addDirtyEntryFlushListener(cacheName, new ThreadCache.DirtyEntryFlushListener() { @Override public void apply(final List<ThreadCache.DirtyEntry> entries) { for (final ThreadCache.DirtyEntry entry : entries) { putAndMaybeForward(entry, context); } } }); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { final String storeName = bytesStore.name(); topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); final Serde<MatchedEvent<K, V>> valueSerDes = new MatchedEventSerde<>( keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.serdes = new StateSerdes<>(topic, new KryoSerDe<Matched>(), valueSerDes); bytesStore.init(context, root); }
public void init(final ProcessorContext context, final StateStore root) { final String storeName = bytesStore.name(); topic = ProcessorStateManager.storeChangelogTopic(context.applicationId(), storeName); bytesStore.init(context, root); Serde<K> kSerde = keySerde == null ? (Serde<K>) context.keySerde() : keySerde; Serde<NFAStates<K, V>> serdes = new NFAStateValueSerde<>( new ComputationStageSerde<>(stages, kSerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde)); this.serdes = new StateSerdes<>(topic, new RunnedKeySerde<>(kSerde), serdes); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { inner.init(context, root); // construct the serde final StateSerdes<K, V> serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.changeLogger = new StoreChangeLogger<>(inner.name(), context, serdes); // if the inner store is an LRU cache, add the eviction listener to log removed record if (inner instanceof MemoryLRUCache) { ((MemoryLRUCache<K, V>) inner).whenEldestRemoved(new MemoryNavigableLRUCache.EldestEntryRemovalListener<K, V>() { @Override public void apply(final K key, final V value) { removed(key); } }); } }
@Override @SuppressWarnings("unchecked") public void init(ProcessorContext context, StateStore root) { inner.init(context, root); // construct the serde StateSerdes<K, V> serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); // Twitter Changed //this.changeLogger = new StoreChangeLogger<>(inner.name(), context, serdes); this.changeLogger = new StoreChangeFlushingLogger<>(inner.name(), context, serdes); // if the inner store is an LRU cache, add the eviction listener to log removed record if (inner instanceof MemoryLRUCache) { ((MemoryLRUCache<K, V>) inner).whenEldestRemoved(new MemoryNavigableLRUCache.EldestEntryRemovalListener<K, V>() { @Override public void apply(K key, V value) { removed(key); } }); } }
@SuppressWarnings("unchecked") private void initInternal(final ProcessorContext context) { this.context = (InternalProcessorContext) context; this.serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), underlying.name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.cache = this.context.getCache(); this.cacheName = ThreadCache.nameSpaceFromTaskIdAndStore(context.taskId().toString(), underlying.name()); cache.addDirtyEntryFlushListener(cacheName, new ThreadCache.DirtyEntryFlushListener() { @Override public void apply(final List<ThreadCache.DirtyEntry> entries) { for (final ThreadCache.DirtyEntry entry : entries) { putAndMaybeForward(entry, (InternalProcessorContext) context); } } }); }
@Override @SuppressWarnings("unchecked") public void init(final ProcessorContext context, final StateStore root) { // construct the serde this.serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), name), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); if (root != null) { // register the store context.register(root, (key, value) -> { // this is a delete if (value == null) { delete(serdes.keyFrom(key)); } else { put(serdes.keyFrom(key), serdes.valueFrom(value)); } }); } this.open = true; }
@SuppressWarnings("unchecked") @Override public void init(final ProcessorContext context, final StateStore root) { this.context = context; this.serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.metrics = (StreamsMetricsImpl) context.metrics(); taskName = context.taskId().toString(); final String metricsGroup = "stream-" + metricScope + "-metrics"; final Map<String, String> taskTags = metrics.tagMap("task-id", taskName, metricScope + "-id", "all"); final Map<String, String> storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name()); putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags); fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags); flushTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "flush", metrics, metricsGroup, taskName, name(), taskTags, storeTags); final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "restore", metrics, metricsGroup, taskName, name(), taskTags, storeTags); // register and possibly restore the state from the logs final long startNs = time.nanoseconds(); try { inner.init(context, root); } finally { this.metrics.recordLatency( restoreTime, startNs, time.nanoseconds() ); } }
@SuppressWarnings("unchecked") @Override public void init(final ProcessorContext context, final StateStore root) { //noinspection unchecked this.serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde, valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde); this.metrics = (StreamsMetricsImpl) context.metrics(); taskName = context.taskId().toString(); final String metricsGroup = "stream-" + metricScope + "-metrics"; final Map<String, String> taskTags = metrics.tagMap("task-id", taskName, metricScope + "-id", "all"); final Map<String, String> storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name()); putTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "put", metrics, metricsGroup, taskName, name(), taskTags, storeTags); fetchTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "fetch", metrics, metricsGroup, taskName, name(), taskTags, storeTags); flushTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "flush", metrics, metricsGroup, taskName, name(), taskTags, storeTags); removeTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "remove", metrics, metricsGroup, taskName, name(), taskTags, storeTags); final Sensor restoreTime = createTaskAndStoreLatencyAndThroughputSensors(DEBUG, "restore", metrics, metricsGroup, taskName, name(), taskTags, storeTags); // register and possibly restore the state from the logs final long startNs = time.nanoseconds(); try { inner.init(context, root); } finally { this.metrics.recordLatency( restoreTime, startNs, time.nanoseconds() ); } }
final Map<String, String> storeTags = metrics.tagMap("task-id", taskName, metricScope + "-id", name()); this.serdes = new StateSerdes<>( ProcessorStateManager.storeChangelogTopic(context.applicationId(), name()), keySerde == null ? (Serde<K>) context.keySerde() : keySerde,