static void verifyStateSupportForWindowingStrategy(WindowingStrategy strategy) { // https://issues.apache.org/jira/browse/BEAM-2507 if (!strategy.getWindowFn().isNonMerging()) { throw new UnsupportedOperationException( String.format( "%s does not currently support state or timers with merging windows", DataflowRunner.class.getSimpleName())); } } }
private <W extends BoundedWindow> WindowingStrategy<?, W> getOutputWindowing( WindowingStrategy<?, W> inputStrategy) { if (inputStrategy.getWindowFn() instanceof InvalidWindows) { @SuppressWarnings("unchecked") InvalidWindows<W> invalidWindows = (InvalidWindows<W>) inputStrategy.getWindowFn(); return inputStrategy.withWindowFn(invalidWindows.getOriginalWindowFn()); } else { return inputStrategy; } } }
public StatefulDoFnRunner( DoFnRunner<InputT, OutputT> doFnRunner, WindowingStrategy<?, ?> windowingStrategy, CleanupTimer cleanupTimer, StateCleaner<W> stateCleaner) { this.doFnRunner = doFnRunner; this.windowingStrategy = windowingStrategy; this.cleanupTimer = cleanupTimer; this.stateCleaner = stateCleaner; WindowFn<?, ?> windowFn = windowingStrategy.getWindowFn(); rejectMergingWindowFn(windowFn); }
public TimeInternalsCleanupTimer( TimerInternals timerInternals, WindowingStrategy<?, ?> windowingStrategy) { this.windowingStrategy = windowingStrategy; WindowFn<?, ?> windowFn = windowingStrategy.getWindowFn(); windowCoder = (Coder<BoundedWindow>) windowFn.windowCoder(); this.timerInternals = timerInternals; }
private static <W extends BoundedWindow> TimerInternals.TimerDataCoder timerDataCoderOf( final WindowingStrategy<?, W> windowingStrategy) { return TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); }
private StateAccessorImpl<K, W> stateAccessor(W window, StateStyle style) { return new StateAccessorImpl<>( activeWindows, windowingStrategy.getWindowFn().windowCoder(), stateInternals, stateContextFromComponents(options, sideInputReader, window), style); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
/** * Returns a {@code PCollectionView<Map<K, V>>} capable of processing elements windowed using the * provided {@link WindowingStrategy}. */ public static <K, V, W extends BoundedWindow> PCollectionView<Map<K, V>> mapView( PCollection<KV<Void, KV<K, V>>> pCollection, WindowingStrategy<?, W> windowingStrategy) { return new SimplePCollectionView<>( pCollection, new MapViewFn<K, V>(), windowingStrategy.getWindowFn().getDefaultWindowMappingFn(), windowingStrategy); }
/** * Returns a {@code PCollectionView<Map<K, Iterable<V>>>} capable of processing elements windowed * using the provided {@link WindowingStrategy}. */ public static <K, V, W extends BoundedWindow> PCollectionView<Map<K, Iterable<V>>> multimapView( PCollection<KV<Void, KV<K, V>>> pCollection, WindowingStrategy<?, W> windowingStrategy) { return new SimplePCollectionView<>( pCollection, new MultimapViewFn<K, V>(), windowingStrategy.getWindowFn().getDefaultWindowMappingFn(), windowingStrategy); }
public ReduceFn<K, InputT, OutputT, W>.OnMergeContext forMerge( Collection<W> activeToBeMerged, W mergeResult, StateStyle style) { return new OnMergeContextImpl( new MergingStateAccessorImpl<>( activeWindows, windowingStrategy.getWindowFn().windowCoder(), stateInternals, style, activeToBeMerged, mergeResult)); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
public ReduceFn<K, InputT, OutputT, W>.OnMergeContext forPremerge(W window) { return new OnPremergeContextImpl( new PremergingStateAccessorImpl<>( activeWindows, windowingStrategy.getWindowFn().windowCoder(), stateInternals, window)); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@Override public void translate(Window.Assign<T> transform, TranslationContext context) { PCollection<T> input = (PCollection<T>) context.getInput(); PCollection<T> output = (PCollection<T>) context.getOutput(); JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input); WindowingStrategy<?, ?> outputStrategy = output.getWindowingStrategy(); WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn(); JavaStream<WindowedValue<T>> outputStream = inputStream.flatMap(new AssignWindows(windowFn), "assign_windows"); context.setOutputStream(output, outputStream); }
@Override public void visitValue(PValue value, Node producer) { if (value instanceof PCollection) { PCollection pc = (PCollection) value; pcollections.add(pc); addCoders(pc.getCoder()); windowingStrategies.add(pc.getWindowingStrategy()); addCoders(pc.getWindowingStrategy().getWindowFn().windowCoder()); } }
@Override public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), KeyedWorkItemCoder.of( GroupByKey.getKeyCoder(input.getCoder()), GroupByKey.getInputValueCoder(input.getCoder()), input.getWindowingStrategy().getWindowFn().windowCoder())); }
@Override public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), KeyedWorkItemCoder.of( GroupByKey.getKeyCoder(input.getCoder()), GroupByKey.getInputValueCoder(input.getCoder()), input.getWindowingStrategy().getWindowFn().windowCoder())); }
@Override public PCollectionTuple expand(PCollection<KV<byte[], KV<InputT, RestrictionT>>> input) { return input .apply(new GBKIntoKeyedWorkItems<>()) .setCoder( KeyedWorkItemCoder.of( ByteArrayCoder.of(), ((KvCoder<byte[], KV<InputT, RestrictionT>>) input.getCoder()).getValueCoder(), input.getWindowingStrategy().getWindowFn().windowCoder())) .apply(new ProcessElements<>(original)); } }
@Test public void testWindowIntoSetWindowfn() { WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10)))) .getWindowingStrategy(); assertTrue(strategy.getWindowFn() instanceof FixedWindows); assertTrue(strategy.getTrigger() instanceof DefaultTrigger); assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, strategy.getMode()); }