@Override public void onData(final T element) { outputCollector.emit(element); }
@Override public void onWatermark(final Watermark inputWatermark) { // If no data, just forwards the watermark if (windowListMap.size() == 0 && currentOutputWatermark < inputWatermark.getTimestamp()) { currentOutputWatermark = inputWatermark.getTimestamp(); outputCollector.emitWatermark(inputWatermark); return; } final Iterator<Map.Entry<BoundedWindow, List<I>>> iterator = windowListMap.entrySet().iterator(); long minOutputTimestampOfEmittedWindows = Long.MAX_VALUE; while (iterator.hasNext()) { final Map.Entry<BoundedWindow, List<I>> entry = iterator.next(); if (entry.getKey().maxTimestamp().getMillis() <= inputWatermark.getTimestamp()) { // emit the windowed data if the watermark timestamp > the window max boundary final O output = viewFn.apply(new MultiView<>(entry.getValue())); outputCollector.emit(WindowedValue.of( output, entry.getKey().maxTimestamp(), entry.getKey(), PaneInfo.ON_TIME_AND_ONLY_FIRING)); iterator.remove(); minOutputTimestampOfEmittedWindows = Math.min(minOutputTimestampOfEmittedWindows, entry.getKey().maxTimestamp().getMillis()); } } if (minOutputTimestampOfEmittedWindows != Long.MAX_VALUE && currentOutputWatermark < minOutputTimestampOfEmittedWindows) { // update current output watermark and emit to next operators currentOutputWatermark = minOutputTimestampOfEmittedWindows; outputCollector.emitWatermark(new Watermark(currentOutputWatermark)); } }
/** * Process a data element down the DAG dependency. */ private void processElement(final OutputCollector outputCollector, final Object dataElement) { outputCollector.emit(dataElement); }
@Override public void close() { outputCollector.emit(aggregatedDynOptData); }
/** * Process a data element down the DAG dependency. */ private void processElement(final OutputCollector outputCollector, final Object dataElement) { outputCollector.emit(dataElement); }
@Override public void onData(final T element) { outputCollector.emit(element); }
@Override public void close() { outputCollector.emit(aggregatedDynOptData); }
@Override public void onData(final T element) { outputCollector.emit(element); }
@Override public void onData(final I element) { try { outputCollector.emit(func.call(element)); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void onData(final T element) { try { Tuple2<K, V> data = func.call(element); outputCollector.emit(data); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void onData(final T element) { if (element == null) { // nothing to be done. return; } try { if (result == null) { result = element; } result = func.call(result, element); } catch (Exception e) { throw new RuntimeException(e); } outputCollector.emit(result); }
@Override public void close() { final Iterator<Map.Entry<K, A>> iterator = keyToAcuumulator.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry<K, A> entry = iterator.next(); final K key = entry.getKey(); final A accum = entry.getValue(); final A compactAccum = combineFnRunner.compact(accum, null, null, null); outputCollector.emit(WindowedValue.valueInGlobalWindow(KV.of(key, compactAccum))); iterator.remove(); // for eager garbage collection } }
@Override public void close() { if (keyToValues.isEmpty()) { LOG.warn("Beam GroupByKeyTransform received no data!"); } else { final Iterator<Map.Entry<Object, List>> iterator = keyToValues.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry<Object, List> entry = iterator.next(); outputCollector.emit(WindowedValue.valueInGlobalWindow(KV.of(entry.getKey(), entry.getValue()))); iterator.remove(); } } } }
@Override public void close() { final Iterator<Map.Entry<K, A>> iterator = keyToAcuumulator.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry<K, A> entry = iterator.next(); final K key = entry.getKey(); final A accum = entry.getValue(); final O output = combineFnRunner.extractOutput(accum, null, null, null); outputCollector.emit(WindowedValue.valueInGlobalWindow(KV.of(key, output))); iterator.remove(); // for eager garbage collection } }
dynOptData.forEach((k, v) -> { final Pair<Object, Object> pairData = Pair.of(k, v); outputCollector.emit(ADDITIONAL_OUTPUT_TAG, pairData); }); return dynOptData;
@Override public void onData(final WindowedValue<T> element) { outputCollector.emit(element.withValue(new SideInputElement<>(index, element.getValue()))); }
dynOptData.forEach((k, v) -> { final Pair<Object, Object> pairData = Pair.of(k, v); outputCollector.emit(ADDITIONAL_OUTPUT_TAG, pairData); }); return dynOptData;
@Override public void onData(final WindowedValue<T> windowedValue) { final BoundedWindow boundedWindow = Iterables.getOnlyElement(windowedValue.getWindows()); final T element = windowedValue.getValue(); final Instant timestamp = windowedValue.getTimestamp(); try { final Collection<W> windows = ((WindowFn<T, W>) windowFn) .assignWindows( ((WindowFn<T, W>) windowFn).new AssignContext() { @Override public T element() { return element; } @Override public Instant timestamp() { return timestamp; } @Override public BoundedWindow window() { return boundedWindow; } }); // Emit compressed windows for efficiency outputCollector.emit(WindowedValue.of(element, timestamp, windows, PaneInfo.NO_FIRING)); } catch (final Exception e) { throw new RuntimeException(e); } }