/** * Applies a window to the input collection if one hasn't already been specified. * * @return the input collection if it already has been windowed, otherwise a the same collection inside a default * window. */ public static <T> PCollection<T> ofDefaultWindow(PCollection<T> in) { if (in.getWindowingStrategy() != WindowingStrategy.globalDefault() && in.getWindowingStrategy() != null) return in; return in.apply("ApplyDefaultWindow", Window.<T> into(FixedWindows.of(DEFAULT_WINDOW_SIZE))); }
/** * Applies a window to the input collection if one hasn't already been specified. * * @return the input collection if it already has been windowed, otherwise a the same collection inside a default * window. */ public static <T> PCollection<T> ofDefaultWindow(PCollection<T> in) { if (in.getWindowingStrategy() != WindowingStrategy.globalDefault() && in.getWindowingStrategy() != null) return in; return in.apply("ApplyDefaultWindow", Window.<T> into(FixedWindows.of(DEFAULT_WINDOW_SIZE))); }
private void verifySupportedTrigger(PCollection<Row> pCollection) { WindowingStrategy windowingStrategy = pCollection.getWindowingStrategy(); if (UNBOUNDED.equals(pCollection.isBounded()) && !triggersOncePerWindow(windowingStrategy)) { throw new UnsupportedOperationException( "Joining unbounded PCollections is currently only supported for " + "non-global windows with triggers that are known to produce output once per window," + "such as the default trigger with zero allowed lateness. " + "In these cases Beam can guarantee it joins all input elements once per window. " + windowingStrategy + " is not supported"); } }
@Override public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, Iterable<InputT>>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder); }
@Override public PCollection<OutputT> expand(PCollection<? extends InputT> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder, onlyOutputTag); }
@Override public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, Iterable<AccumT>>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder); }
@Override public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder()); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
@Override public PCollection<ElemT> expand(PCollection<ElemT> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder()); }
private PCollectionContentsAssert<T> withPane( BoundedWindow window, SimpleFunction<Iterable<ValueInSingleWindow<T>>, Iterable<T>> paneExtractor) { @SuppressWarnings({"unchecked", "rawtypes"}) Coder<BoundedWindow> windowCoder = (Coder) actual.getWindowingStrategy().getWindowFn().windowCoder(); return new PCollectionContentsAssert<>( actual, IntoStaticWindows.of(windowCoder, window), paneExtractor, site); }
@Override @SuppressWarnings("deprecation") public PCollection<Iterable<ElemT>> expand(PCollection<Iterable<ElemT>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder()); }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public PCollection<KV<K, Iterable<WindowedValue<V>>>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), (Coder) GroupByKey.getOutputKvCoder(input.getCoder())); } }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@Override public PCollection<KV<K, Iterable<InputT>>> expand(PCollection<KV<K, InputT>> input) { Duration allowedLateness = input.getWindowingStrategy().getAllowedLateness(); checkArgument( input.getCoder() instanceof KvCoder, "coder specified in the input PCollection is not a KvCoder"); KvCoder inputCoder = (KvCoder) input.getCoder(); Coder<K> keyCoder = (Coder<K>) inputCoder.getCoderArguments().get(0); Coder<InputT> valueCoder = (Coder<InputT>) inputCoder.getCoderArguments().get(1); return input.apply( ParDo.of(new GroupIntoBatchesDoFn<>(batchSize, allowedLateness, keyCoder, valueCoder))); }
@Override public void translate(Window.Assign<T> transform, TranslationContext context) { PCollection<T> input = (PCollection<T>) context.getInput(); PCollection<T> output = (PCollection<T>) context.getOutput(); JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input); WindowingStrategy<?, ?> outputStrategy = output.getWindowingStrategy(); WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn(); JavaStream<WindowedValue<T>> outputStream = inputStream.flatMap(new AssignWindows(windowFn), "assign_windows"); context.setOutputStream(output, outputStream); }
@Override public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), KeyedWorkItemCoder.of( GroupByKey.getKeyCoder(input.getCoder()), GroupByKey.getInputValueCoder(input.getCoder()), input.getWindowingStrategy().getWindowFn().windowCoder())); }
@Override public PCollection<KeyedWorkItem<K, V>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), KeyedWorkItemCoder.of( GroupByKey.getKeyCoder(input.getCoder()), GroupByKey.getInputValueCoder(input.getCoder()), input.getWindowingStrategy().getWindowFn().windowCoder())); }
@Test @Category(ValidatesRunner.class) public void testJustReshuffle() { PCollection<KV<String, Integer>> input = pipeline.apply( Create.of(ARBITRARY_KVS).withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))); PCollection<KV<String, Integer>> output = input.apply(Reshuffle.of()); PAssert.that(output).containsInAnyOrder(ARBITRARY_KVS); assertEquals(input.getWindowingStrategy(), output.getWindowingStrategy()); pipeline.run(); }
@Test public void testWindowIntoSetWindowfn() { WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10)))) .getWindowingStrategy(); assertTrue(strategy.getWindowFn() instanceof FixedWindows); assertTrue(strategy.getTrigger() instanceof DefaultTrigger); assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, strategy.getMode()); }