StreamKafkaP( @Nonnull Properties properties, @Nonnull List<String> topics, @Nonnull DistributedFunction<? super ConsumerRecord<K, V>, ? extends T> projectionFn, @Nonnull EventTimePolicy<? super T> eventTimePolicy ) { this.properties = properties; this.topics = topics; this.projectionFn = projectionFn; eventTimeMapper = new EventTimeMapper<>(eventTimePolicy); partitionCounts = new int[topics.size()]; }
public InsertWatermarksP(EventTimePolicy<? super T> eventTimePolicy) { eventTimeMapper = new EventTimeMapper<>(eventTimePolicy); eventTimeMapper.increasePartitionCount(1); }
StreamJmsP(Connection connection, DistributedFunction<? super Connection, ? extends Session> sessionFn, DistributedFunction<? super Session, ? extends MessageConsumer> consumerFn, DistributedConsumer<? super Session> flushFn, DistributedFunction<? super Message, ? extends T> projectionFn, EventTimePolicy<? super T> eventTimePolicy ) { this.connection = connection; this.sessionFn = sessionFn; this.consumerFn = consumerFn; this.flushFn = flushFn; this.projectionFn = projectionFn; eventTimeMapper = new EventTimeMapper<>(eventTimePolicy); eventTimeMapper.increasePartitionCount(1); }
public ConvenientSourceP( @Nonnull Function<? super Context, ? extends S> createFn, @Nonnull BiConsumer<? super S, ? super SourceBufferConsumerSide<?>> fillBufferFn, @Nonnull Consumer<? super S> destroyFn, @Nonnull SourceBufferConsumerSide<?> buffer, @Nullable EventTimePolicy<? super T> eventTimePolicy ) { this.createFn = createFn; this.fillBufferFn = fillBufferFn; this.destroyFn = destroyFn; this.buffer = buffer; if (eventTimePolicy != null) { eventTimeMapper = new EventTimeMapper<>(eventTimePolicy); eventTimeMapper.increasePartitionCount(1); } else { eventTimeMapper = null; } }
StreamEventJournalP( @Nonnull EventJournalReader<? extends E> eventJournalReader, @Nonnull List<Integer> assignedPartitions, @Nonnull DistributedPredicate<? super E> predicateFn, @Nonnull DistributedFunction<? super E, ? extends T> projectionFn, @Nonnull JournalInitialPosition initialPos, boolean isRemoteReader, @Nonnull EventTimePolicy<? super T> eventTimePolicy ) { this.eventJournalReader = eventJournalReader; this.predicate = (Predicate<? super E> & Serializable) predicateFn::test; this.projection = toProjection(projectionFn); this.initialPos = initialPos; this.isRemoteReader = isRemoteReader; partitionIds = assignedPartitions.stream().mapToInt(Integer::intValue).toArray(); emitOffsets = new long[partitionIds.length]; readOffsets = new long[partitionIds.length]; eventTimeMapper = new EventTimeMapper<>(eventTimePolicy); // Do not coalesce partition WMs because the number of partitions is far // larger than the number of consumers by default and it is not // configurable on a per journal basis. This creates excessive latency // when the number of events are relatively low and we have to wait for // all partitions to advance before advancing the watermark. The side // effect of not coalescing is that when the job is restarted and catching // up, there might be dropped late events due to several events being read // from one partition before the rest and the partition advancing ahead of // others. This might be changed in the future and/or made optional. assert partitionIds.length > 0 : "no partitions assigned"; eventTimeMapper.increasePartitionCount(1); }