@Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return window.maxTimestamp(); }
/** * Return {@code timestamp}, possibly shifted forward in time according to the window strategy's * output time function. */ private Instant shift(Instant timestamp, W window) { Instant shifted = windowingStrategy .getTimestampCombiner() .assign(window, windowingStrategy.getWindowFn().getOutputTime(timestamp, window)); // Don't call checkState(), to avoid calling BoundedWindow.formatTimestamp() every time if (shifted.isBefore(timestamp)) { throw new IllegalStateException( String.format( "TimestampCombiner moved element from %s to earlier time %s for window %s", BoundedWindow.formatTimestamp(timestamp), BoundedWindow.formatTimestamp(shifted), window)); } checkState( timestamp.isAfter(window.maxTimestamp()) || !shifted.isAfter(window.maxTimestamp()), "TimestampCombiner moved element from %s to %s which is beyond end of " + "window %s", timestamp, shifted, window); return shifted; }
@Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { return window.maxTimestamp(); }
@Override public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) { // Even without an early or late trigger, we'll still produce a firing at the watermark. return window.maxTimestamp(); }
@Override public Instant merge(BoundedWindow intoWindow, Iterable<? extends Instant> mergingTimestamps) { return intoWindow.maxTimestamp(); }
private static Instant assignOutputTime( TimestampCombiner timestampCombiner, Instant inputTimestamp, BoundedWindow window) { switch (timestampCombiner) { case EARLIEST: case LATEST: return inputTimestamp; case END_OF_WINDOW: return window.maxTimestamp(); default: throw new IllegalArgumentException( String.format("Unknown %s: %s", TimestampCombiner.class, timestampCombiner)); } }
@Override public W getSideInputWindow(BoundedWindow mainWindow) { if (mainWindow instanceof GlobalWindow) { throw new IllegalArgumentException( "Attempted to get side input window for GlobalWindow from non-global WindowFn"); } return assignWindow(mainWindow.maxTimestamp()); } };
@FinishBundle public void finishBundle(FinishBundleContext c) { for (BoundedWindow w : windows) { c.output(null, w.maxTimestamp(), w); } } }
static <K, V> void output(DoFn<?, KV<K, V>>.FinishBundleContext context, Map<? extends K, ? extends V> map, BoundedWindow window) { for (Map.Entry<? extends K, ? extends V> entry : map.entrySet()) { context.output(KV.<K, V>of(entry.getKey(), entry.getValue()), window.maxTimestamp(), window); } }
static <K, V> void output(DoFn<?, KV<K, V>>.FinishBundleContext context, Map<? extends K, ? extends V> map, BoundedWindow window) { for (Map.Entry<? extends K, ? extends V> entry : map.entrySet()) { context.output(KV.<K, V>of(entry.getKey(), entry.getValue()), window.maxTimestamp(), window); } }
@Override public boolean isReady(final PCollectionView view, final BoundedWindow window) { return window.maxTimestamp().getMillis() < curWatermark || inMemorySideInputs.containsKey(Pair.of(view, window)); }
/** * Say a DoFn of this reader has 3 main inputs and 4 side inputs. * {@link org.apache.nemo.runtime.executor.datatransfer.InputWatermarkManager} guarantees that the watermark here * is the minimum of the all 7 input streams. * @param newWatermark to set. */ public void setCurrentWatermarkOfAllMainAndSideInputs(final long newWatermark) { if (curWatermark > newWatermark) { // Cannot go backwards in time. throw new IllegalStateException(curWatermark + " > " + newWatermark); } this.curWatermark = newWatermark; // TODO #282: Handle late data inMemorySideInputs.entrySet().removeIf(entry -> { return entry.getKey().right().maxTimestamp().getMillis() <= this.curWatermark; // Discard old sideinputs. }); } }
protected static <T> Iterable<WindowedValue<T>> sortByWindows(Iterable<WindowedValue<T>> iter) { List<WindowedValue<T>> sorted = Lists.newArrayList(iter); sorted.sort(Comparator.comparing(o -> Iterables.getOnlyElement(o.getWindows()).maxTimestamp())); return sorted; }
@Override public IntervalWindow getSideInputWindow(BoundedWindow mainWindow) { if (mainWindow instanceof GlobalWindow) { throw new IllegalArgumentException( "Attempted to get side input window for GlobalWindow from non-global WindowFn"); } long lastStart = lastStartFor(mainWindow.maxTimestamp().minus(size)); return new IntervalWindow(new Instant(lastStart + period.getMillis()), size); } };
@Override public int compare( WindowedValue<KV<K, InputT>> o1, WindowedValue<KV<K, InputT>> o2) { return Iterables.getOnlyElement(o1.getWindows()).maxTimestamp() .compareTo(Iterables.getOnlyElement(o2.getWindows()).maxTimestamp()); } });
c.output( new Result<>(result.resourceId.toString(), result.byteSize, destination), writerWindows.get(destination).maxTimestamp(), writerWindows.get(destination)); } catch (Exception e) {
@Override public void onElement(OnElementContext c) throws Exception { // If the end of the window has already been reached, then we are already ready to fire // and do not need to set a wake-up timer. if (!endOfWindowReached(c)) { c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); } }
@Override public void onMerge(OnMergeContext c) throws Exception { // If the end of the window has already been reached, then we are already ready to fire // and do not need to set a wake-up timer. if (!endOfWindowReached(c)) { c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); } }
@Override public void onElement(OnElementContext c) throws Exception { // We're interested in knowing when the input watermark passes the end of the window. // (It is possible this has already happened, in which case the timer will be fired // almost immediately). if (!endOfWindowReached(c)) { c.setTimer(c.window().maxTimestamp(), TimeDomain.EVENT_TIME); } }
@ProcessElement public void processElement(ProcessContext c, BoundedWindow window) throws Exception { c.output(KV.of(window.maxTimestamp(), c.element())); } }