/** * Returns an event time policy that results in no timestamping. Only * useful in jobs with streaming sources that don't do any aggregation. * If there is an aggregation step in the job and you use these parameters, * your job will keep accumulating the data without producing any output. */ public static <T> EventTimePolicy<T> noEventTime() { return eventTimePolicy(i -> Long.MIN_VALUE, noWrapping(), NO_WATERMARKS, 0, 0, -1); }
/** * Creates and returns a new event time policy. To get a policy that * results in no watermarks being emitted, call {@link * #noEventTime()}. * * @param timestampFn function that extracts the timestamp from the event; * if null, Jet will use the source's native timestamp * @param newWmPolicyFn factory of the watermark policy objects * @param watermarkThrottlingFrameSize the frame length to which we * throttle watermarks, see {@link #watermarkThrottlingFrameSize()} * @param watermarkThrottlingFrameOffset the frame offset to which we * throttle watermarks, see {@link #watermarkThrottlingFrameOffset()} * @param idleTimeoutMillis the timeout after which a partition will be * marked as <em>idle</em>. */ public static <T> EventTimePolicy<T> eventTimePolicy( @Nullable DistributedToLongFunction<? super T> timestampFn, @Nonnull DistributedSupplier<? extends WatermarkPolicy> newWmPolicyFn, long watermarkThrottlingFrameSize, long watermarkThrottlingFrameOffset, long idleTimeoutMillis ) { return eventTimePolicy(timestampFn, noWrapping(), newWmPolicyFn, watermarkThrottlingFrameSize, watermarkThrottlingFrameOffset, idleTimeoutMillis); }
/** * Returns a new instance with emit policy replaced with the given * argument. */ @Nonnull private static <T> EventTimePolicy<T> withFrameSize( EventTimePolicy<T> original, long watermarkThrottlingFrameSize ) { return eventTimePolicy(original.timestampFn(), original.wrapFn(), original.newWmPolicyFn(), watermarkThrottlingFrameSize, 0, original.idleTimeoutMillis()); }
@SuppressWarnings("unchecked") StreamStage<T> addTimestampsInt( @Nullable DistributedToLongFunction<? super T> timestampFn, long allowedLateness, boolean tryAddToSource) { checkSerializable(timestampFn, "timestampFn"); checkFalse(hasJetEvents(), "This stage already has timestamps assigned to it"); DistributedSupplier<WatermarkPolicy> wmPolicy = limitingLag(allowedLateness); EventTimePolicy<T> eventTimePolicy = eventTimePolicy( timestampFn, JetEvent::jetEvent, wmPolicy, 0, 0, DEFAULT_IDLE_TIMEOUT ); if (tryAddToSource && transform instanceof StreamSourceTransform) { ((StreamSourceTransform<T>) transform).setEventTimePolicy(eventTimePolicy); this.fnAdapter = ADAPT_TO_JET_EVENT; return (StreamStage<T>) this; } TimestampTransform<T> tsTransform = new TimestampTransform<>(transform, eventTimePolicy); pipelineImpl.connect(transform, tsTransform); return new StreamStageImpl<>(tsTransform, ADAPT_TO_JET_EVENT, pipelineImpl); }