public WatermarkHold(TimerInternals timerInternals, WindowingStrategy<?, W> windowingStrategy) { this.timerInternals = timerInternals; this.windowingStrategy = windowingStrategy; this.elementHoldTag = watermarkHoldTagForTimestampCombiner(windowingStrategy.getTimestampCombiner()); }
@SafeVarargs public final void assertHasOnlyGlobalAndFinishedSetsAndPaneInfoFor(W... expectedWindows) { assertHasOnlyGlobalAndAllowedTags( ImmutableSet.copyOf(expectedWindows), ImmutableSet.of( TriggerStateMachineRunner.FINISHED_BITS_TAG, PaneInfoTracker.PANE_INFO_TAG, WatermarkHold.watermarkHoldTagForTimestampCombiner( objectStrategy.getTimestampCombiner()), WatermarkHold.EXTRA_HOLD_TAG)); }
@SafeVarargs public final void assertHasOnlyGlobalAndPaneInfoFor(W... expectedWindows) { assertHasOnlyGlobalAndAllowedTags( ImmutableSet.copyOf(expectedWindows), ImmutableSet.of( PaneInfoTracker.PANE_INFO_TAG, WatermarkHold.watermarkHoldTagForTimestampCombiner( objectStrategy.getTimestampCombiner()), WatermarkHold.EXTRA_HOLD_TAG)); }
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
@SafeVarargs public final void assertHasOnlyGlobalAndStateFor(W... expectedWindows) { assertHasOnlyGlobalAndAllowedTags( ImmutableSet.copyOf(expectedWindows), ImmutableSet.of( ((SystemReduceFn<?, ?, ?, ?, ?>) reduceFn).getBufferTag(), TriggerStateMachineRunner.FINISHED_BITS_TAG, PaneInfoTracker.PANE_INFO_TAG, WatermarkHold.watermarkHoldTagForTimestampCombiner( objectStrategy.getTimestampCombiner()), WatermarkHold.EXTRA_HOLD_TAG)); }
/** * Return {@code timestamp}, possibly shifted forward in time according to the window strategy's * output time function. */ private Instant shift(Instant timestamp, W window) { Instant shifted = windowingStrategy .getTimestampCombiner() .assign(window, windowingStrategy.getWindowFn().getOutputTime(timestamp, window)); // Don't call checkState(), to avoid calling BoundedWindow.formatTimestamp() every time if (shifted.isBefore(timestamp)) { throw new IllegalStateException( String.format( "TimestampCombiner moved element from %s to earlier time %s for window %s", BoundedWindow.formatTimestamp(timestamp), BoundedWindow.formatTimestamp(shifted), window)); } checkState( timestamp.isAfter(window.maxTimestamp()) || !shifted.isAfter(window.maxTimestamp()), "TimestampCombiner moved element from %s to %s which is beyond end of " + "window %s", timestamp, shifted, window); return shifted; }
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner();
new CombineInputs<>( combineFn, input.getWindowingStrategy().getTimestampCombiner(), inputCoder.getKeyCoder()))) .setCoder(KvCoder.of(inputCoder.getKeyCoder(), accumulatorCoder))
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner();
@Override public boolean equals(Object object) { if (!(object instanceof WindowingStrategy)) { return false; } WindowingStrategy<?, ?> other = (WindowingStrategy<?, ?>) object; return isAllowedLatenessSpecified() == other.isAllowedLatenessSpecified() && isModeSpecified() == other.isModeSpecified() && isTimestampCombinerSpecified() == other.isTimestampCombinerSpecified() && getMode().equals(other.getMode()) && getAllowedLateness().equals(other.getAllowedLateness()) && getClosingBehavior().equals(other.getClosingBehavior()) && getOnTimeBehavior().equals(other.getOnTimeBehavior()) && getTrigger().equals(other.getTrigger()) && getTimestampCombiner().equals(other.getTimestampCombiner()) && getWindowFn().equals(other.getWindowFn()); }
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner(); WindowFn<?, BoundedWindow> windowFn = windowingStrategy.getWindowFn();
(TimestampCombiner) windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
(TimestampCombiner) windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
(TimestampCombiner) windowingStrategy.getTimestampCombiner(); WindowFn<Object, W> windowFn = windowingStrategy.getWindowFn();
@Override public void translate(GroupByKey<K, V> transform, TranslationContext context) { PCollection<KV<K, V>> input = (PCollection<KV<K, V>>) context.getInput(); Coder<K> inputKeyCoder = ((KvCoder<K, V>) input.getCoder()).getKeyCoder(); JavaStream<WindowedValue<KV<K, V>>> inputStream = context.getInputStream(input); int parallelism = context.getPipelineOptions().getParallelism(); TimestampCombiner timestampCombiner = input.getWindowingStrategy().getTimestampCombiner(); WindowFn<KV<K, V>, BoundedWindow> windowFn = (WindowFn<KV<K, V>, BoundedWindow>) input.getWindowingStrategy().getWindowFn(); JavaStream<WindowedValue<KV<K, List<V>>>> outputStream = inputStream .window( Windows.apply( new GearpumpWindowFn(windowFn.isNonMerging()), EventTimeTrigger$.MODULE$, Discarding$.MODULE$, windowFn.toString())) .groupBy(new GroupByFn<>(inputKeyCoder), parallelism, "group_by_Key_and_Window") .map(new KeyedByTimestamp<>(windowFn, timestampCombiner), "keyed_by_timestamp") .fold(new Merge<>(windowFn, timestampCombiner), "merge") .map(new Values<>(), "values"); context.setOutputStream(context.getOutput(), outputStream); }
TimestampCombiner timestampCombiner = windowingStrategy.getTimestampCombiner(); WindowFn<?, BoundedWindow> windowFn = windowingStrategy.getWindowFn();
/** * Converts a {@link WindowingStrategy} into a {@link RunnerApi.WindowingStrategy}, registering * any components in the provided {@link SdkComponents}. */ public static RunnerApi.WindowingStrategy toProto( WindowingStrategy<?, ?> windowingStrategy, SdkComponents components) throws IOException { SdkFunctionSpec windowFnSpec = toProto(windowingStrategy.getWindowFn(), components); RunnerApi.WindowingStrategy.Builder windowingStrategyProto = RunnerApi.WindowingStrategy.newBuilder() .setOutputTime(toProto(windowingStrategy.getTimestampCombiner())) .setAccumulationMode(toProto(windowingStrategy.getMode())) .setClosingBehavior(toProto(windowingStrategy.getClosingBehavior())) .setAllowedLateness(windowingStrategy.getAllowedLateness().getMillis()) .setTrigger(TriggerTranslation.toProto(windowingStrategy.getTrigger())) .setWindowFn(windowFnSpec) .setAssignsToOneWindow(windowingStrategy.getWindowFn().assignsToOneWindow()) .setOnTimeBehavior(toProto(windowingStrategy.getOnTimeBehavior())) .setWindowCoderId( components.registerCoder(windowingStrategy.getWindowFn().windowCoder())); return windowingStrategyProto.build(); }