/** * Returns a new instance with emit policy replaced with the given * argument. */ @Nonnull private static <T> EventTimePolicy<T> withFrameSize( EventTimePolicy<T> original, long watermarkThrottlingFrameSize ) { return eventTimePolicy(original.timestampFn(), original.wrapFn(), original.newWmPolicyFn(), watermarkThrottlingFrameSize, 0, original.idleTimeoutMillis()); }
/** * Returns an event time policy that results in no timestamping. Only * useful in jobs with streaming sources that don't do any aggregation. * If there is an aggregation step in the job and you use these parameters, * your job will keep accumulating the data without producing any output. */ public static <T> EventTimePolicy<T> noEventTime() { return eventTimePolicy(i -> Long.MIN_VALUE, noWrapping(), NO_WATERMARKS, 0, 0, -1); }
/** * The partition count is initially set to 0, call * {@link #increasePartitionCount} to set it. * * @param eventTimePolicy event time policy as passed in {@link * Sources#streamFromProcessorWithWatermarks} **/ public EventTimeMapper(EventTimePolicy<? super T> eventTimePolicy) { this.idleTimeoutNanos = MILLISECONDS.toNanos(eventTimePolicy.idleTimeoutMillis()); this.timestampFn = eventTimePolicy.timestampFn(); this.wrapFn = eventTimePolicy.wrapFn(); this.newWmPolicyFn = eventTimePolicy.newWmPolicyFn(); if (eventTimePolicy.watermarkThrottlingFrameSize() != 0) { this.watermarkThrottlingFrame = tumblingWinPolicy(eventTimePolicy.watermarkThrottlingFrameSize()) .withOffset(eventTimePolicy.watermarkThrottlingFrameOffset()); } else { this.watermarkThrottlingFrame = null; } }
public TimestampTransform( @Nonnull Transform upstream, @Nonnull EventTimePolicy<? super T> eventTimePolicy ) { super("add-timestamps", upstream); this.eventTimePolicy = eventTimePolicy; checkNotNull(eventTimePolicy.timestampFn(), "timestampFn must not be null if timestamps aren't added in the source"); }
/** * Creates and returns a new event time policy. To get a policy that * results in no timestamping, call {@link #noEventTime()}. * * @param timestampFn function that extracts the timestamp from the event; * if null, Jet will use the source's native timestamp * @param wrapFn function that transforms the received item and its * timestamp into the emitted item * @param newWmPolicyFn factory of the watermark policy objects * @param watermarkThrottlingFrameSize the frame length to which we * throttle watermarks, see {@link #watermarkThrottlingFrameSize()} * @param watermarkThrottlingFrameOffset the frame offset to which we * throttle watermarks, see {@link #watermarkThrottlingFrameOffset()} * @param idleTimeoutMillis the timeout after which a partition will be * marked as <em>idle</em>. Use 0 to disable the feature. */ public static <T> EventTimePolicy<T> eventTimePolicy( @Nullable DistributedToLongFunction<? super T> timestampFn, @Nonnull DistributedObjLongBiFunction<? super T, ?> wrapFn, @Nonnull DistributedSupplier<? extends WatermarkPolicy> newWmPolicyFn, long watermarkThrottlingFrameSize, long watermarkThrottlingFrameOffset, long idleTimeoutMillis ) { return new EventTimePolicy<>(timestampFn, wrapFn, newWmPolicyFn, watermarkThrottlingFrameSize, watermarkThrottlingFrameOffset, idleTimeoutMillis); }
@SuppressWarnings("unchecked") StreamStage<T> addTimestampsInt( @Nullable DistributedToLongFunction<? super T> timestampFn, long allowedLateness, boolean tryAddToSource) { checkSerializable(timestampFn, "timestampFn"); checkFalse(hasJetEvents(), "This stage already has timestamps assigned to it"); DistributedSupplier<WatermarkPolicy> wmPolicy = limitingLag(allowedLateness); EventTimePolicy<T> eventTimePolicy = eventTimePolicy( timestampFn, JetEvent::jetEvent, wmPolicy, 0, 0, DEFAULT_IDLE_TIMEOUT ); if (tryAddToSource && transform instanceof StreamSourceTransform) { ((StreamSourceTransform<T>) transform).setEventTimePolicy(eventTimePolicy); this.fnAdapter = ADAPT_TO_JET_EVENT; return (StreamStage<T>) this; } TimestampTransform<T> tsTransform = new TimestampTransform<>(transform, eventTimePolicy); pipelineImpl.connect(transform, tsTransform); return new StreamStageImpl<>(tsTransform, ADAPT_TO_JET_EVENT, pipelineImpl); }
@Override @SuppressWarnings("unchecked") public void addToDag(Planner p) { if (emitsWatermarks || eventTimePolicy == null) { // Reached when the source either emits both JetEvents and watermarks // or neither. In these cases we don't have to insert watermarks. p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), metaSupplierFn.apply(eventTimePolicy != null ? eventTimePolicy : noEventTime()) ); } else { // ------------ // | sourceP | // ------------ // | // isolated // v // ------------- // | insertWmP | // ------------- String v1name = p.uniqueVertexName(name()); Vertex v1 = p.dag.newVertex(v1name, metaSupplierFn.apply(eventTimePolicy)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex( this, v1name + "-add-timestamps", localParallelism(), insertWatermarksP(eventTimePolicy) ); p.dag.edge(between(v1, pv2.v).isolated()); } }
/** * Creates and returns a new event time policy. To get a policy that * results in no watermarks being emitted, call {@link * #noEventTime()}. * * @param timestampFn function that extracts the timestamp from the event; * if null, Jet will use the source's native timestamp * @param newWmPolicyFn factory of the watermark policy objects * @param watermarkThrottlingFrameSize the frame length to which we * throttle watermarks, see {@link #watermarkThrottlingFrameSize()} * @param watermarkThrottlingFrameOffset the frame offset to which we * throttle watermarks, see {@link #watermarkThrottlingFrameOffset()} * @param idleTimeoutMillis the timeout after which a partition will be * marked as <em>idle</em>. */ public static <T> EventTimePolicy<T> eventTimePolicy( @Nullable DistributedToLongFunction<? super T> timestampFn, @Nonnull DistributedSupplier<? extends WatermarkPolicy> newWmPolicyFn, long watermarkThrottlingFrameSize, long watermarkThrottlingFrameOffset, long idleTimeoutMillis ) { return eventTimePolicy(timestampFn, noWrapping(), newWmPolicyFn, watermarkThrottlingFrameSize, watermarkThrottlingFrameOffset, idleTimeoutMillis); }