@SystemDoFnInternal private static class GroupByKeyHashAndSortByKeyAndWindowDoFn<K, V, W> extends DoFn<KV<K, V>, KV<Integer, KV<KV<K, W>, WindowedValue<V>>>> { private final IsmRecordCoder<?> coder; private GroupByKeyHashAndSortByKeyAndWindowDoFn(IsmRecordCoder<?> coder) { this.coder = coder; } @ProcessElement public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception { @SuppressWarnings("unchecked") W window = (W) untypedWindow; c.output( KV.of( coder.hash(ImmutableList.of(c.element().getKey())), KV.of( KV.of(c.element().getKey(), window), WindowedValue.of( c.element().getValue(), c.timestamp(), untypedWindow, c.pane())))); } }
/** * A {@link DoFn} that for each element outputs a {@code KV} structure suitable for grouping by * the hash of the window's byte representation and sorting the grouped values using the * window's byte representation. */ @SystemDoFnInternal private static class UseWindowHashAsKeyAndWindowAsSortKeyDoFn<T, W extends BoundedWindow> extends DoFn<T, KV<Integer, KV<W, WindowedValue<T>>>> { private final IsmRecordCoder<?> ismCoderForHash; private UseWindowHashAsKeyAndWindowAsSortKeyDoFn(IsmRecordCoder<?> ismCoderForHash) { this.ismCoderForHash = ismCoderForHash; } @ProcessElement public void processElement(ProcessContext c, BoundedWindow untypedWindow) throws Exception { @SuppressWarnings("unchecked") W window = (W) untypedWindow; c.output( KV.of( ismCoderForHash.hash(ImmutableList.of(window)), KV.of(window, WindowedValue.of(c.element(), c.timestamp(), window, c.pane())))); } }
/** * A {@link DoFn} which creates {@link IsmRecord}s assuming that each element is within the * global window. Each {@link IsmRecord} has * * <ul> * <li>Key 1: Global window * <li>Key 2: Index offset within window * <li>Value: Windowed value * </ul> */ @SystemDoFnInternal static class ToIsmRecordForGlobalWindowDoFn<T> extends DoFn<T, IsmRecord<WindowedValue<T>>> { long indexInBundle; @StartBundle public void startBundle() throws Exception { indexInBundle = 0; } @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output( IsmRecord.of( ImmutableList.of(GlobalWindow.INSTANCE, indexInBundle), WindowedValue.of(c.element(), c.timestamp(), GlobalWindow.INSTANCE, c.pane()))); indexInBundle += 1; } }
@SystemDoFnInternal static class ToIsmRecordForNonGlobalWindowDoFn<T, W extends BoundedWindow> extends DoFn<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>, IsmRecord<WindowedValue<T>>> {
@SystemDoFnInternal public class GroupAlsoByWindowViaWindowSetNewDoFn< K, InputT, OutputT, W extends BoundedWindow, RinT extends KeyedWorkItem<K, InputT>>
@SystemDoFnInternal @VisibleForTesting class StreamingWriteFn<ErrorT> extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {