@Test public void testDeleteTimer() { TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); String key1 = "key1"; Instant instant0 = new Instant(0); Instant instant1 = new Instant(1); ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder); timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now(), null); TimerData timerData0 = TimerData.of("timerData0", StateNamespaces.global(), instant0, TimeDomain.EVENT_TIME); timerInternals.setTimer(timerData0); TimerData timerData1 = TimerData.of("timerData1", StateNamespaces.global(), instant1, TimeDomain.EVENT_TIME); timerInternals.setTimer(timerData1); Map<?, Set<Slice>> timerMap = timerInternals.getTimerSet(TimeDomain.EVENT_TIME).getMap(); assertEquals(1, timerMap.size()); assertEquals(2, timerMap.values().iterator().next().size()); timerInternals.deleteTimer(timerData0.getNamespace(), timerData0.getTimerId()); assertEquals(1, timerMap.size()); assertEquals(1, timerMap.values().iterator().next().size()); timerInternals.deleteTimer( timerData1.getNamespace(), timerData1.getTimerId(), TimeDomain.PROCESSING_TIME); assertEquals(1, timerMap.size()); assertEquals(1, timerMap.values().iterator().next().size()); timerInternals.deleteTimer( timerData1.getNamespace(), timerData1.getTimerId(), TimeDomain.EVENT_TIME); assertEquals(0, timerMap.size()); }
TimerProcessor<String> timerProcessor = firedTimers::put; TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); String key1 = "key1"; Instant instant0 = new Instant(0);
TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); this.currentKeyTimerInternals = new ApexTimerInternals<>(timerCoder);
TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder());
public DoFnOperator( DoFn<InputT, OutputT> doFn, String stepName, Coder<WindowedValue<InputT>> inputCoder, TupleTag<OutputT> mainOutputTag, List<TupleTag<?>> additionalOutputTags, OutputManagerFactory<OutputT> outputManagerFactory, WindowingStrategy<?, ?> windowingStrategy, Map<Integer, PCollectionView<?>> sideInputTagMapping, Collection<PCollectionView<?>> sideInputs, PipelineOptions options, Coder<?> keyCoder) { this.doFn = doFn; this.stepName = stepName; this.inputCoder = inputCoder; this.mainOutputTag = mainOutputTag; this.additionalOutputTags = additionalOutputTags; this.sideInputTagMapping = sideInputTagMapping; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(options); this.windowingStrategy = windowingStrategy; this.outputManagerFactory = outputManagerFactory; setChainingStrategy(ChainingStrategy.ALWAYS); this.keyCoder = keyCoder; this.timerCoder = TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); FlinkPipelineOptions flinkOptions = options.as(FlinkPipelineOptions.class); this.maxBundleSize = flinkOptions.getMaxBundleSize(); this.maxBundleTimeMills = flinkOptions.getMaxBundleTimeMills(); }
@Test public void testSerialization() { TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE); TimerData timerData = TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME); String key = "key"; ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder); timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now(), null); timerInternals.setTimer(timerData); ApexTimerInternals<String> cloned; assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(timerInternals)); cloned.setContext(key, StringUtf8Coder.of(), Instant.now(), null); Map<?, Set<Slice>> timers = cloned.getTimerSet(TimeDomain.EVENT_TIME).getMap(); assertEquals(1, timers.size()); } }
@Test public void testTimerDataCoder() throws Exception { CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(GlobalWindow.Coder.INSTANCE), TimerData.of( "arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME)); Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder(); CoderProperties.coderDecodeEncodeEqual( TimerDataCoder.of(windowCoder), TimerData.of( "another-id", StateNamespaces.window( windowCoder, new IntervalWindow(new Instant(0), new Instant(100))), new Instant(99), TimeDomain.PROCESSING_TIME)); }
@SuppressWarnings("unchecked") public ApexGroupByKeyOperator( ApexPipelineOptions pipelineOptions, PCollection<KV<K, V>> input, ApexStateBackend stateBackend) { checkNotNull(pipelineOptions); this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); this.windowingStrategy = (WindowingStrategy<V, BoundedWindow>) input.getWindowingStrategy(); this.keyCoder = ((KvCoder<K, V>) input.getCoder()).getKeyCoder(); this.valueCoder = ((KvCoder<K, V>) input.getCoder()).getValueCoder(); this.stateInternalsFactory = stateBackend.newStateInternalsFactory(keyCoder); TimerInternals.TimerDataCoder timerCoder = TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); this.timerInternals = new ApexTimerInternals<>(timerCoder); }
@Test public void testCoderIsSerializableWithWellKnownCoderType() { CoderProperties.coderSerializable(TimerDataCoder.of(GlobalWindow.Coder.INSTANCE)); }
private static <W extends BoundedWindow> TimerInternals.TimerDataCoder timerDataCoderOf( final WindowingStrategy<?, W> windowingStrategy) { return TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); }
private KeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; this.timersCoder = IterableCoder.of(TimerDataCoder.of(windowCoder)); this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder)); }