@Override public boolean shutdownServiceUninterruptible(long timeoutMs) { shutdownService(); return true; }
@Override protected ProcessingTimeService getProcessingTimeService() { if (testProcessingTimeService == null) { testProcessingTimeService = new TestProcessingTimeService(); } return testProcessingTimeService; } }
@Override public void run(SourceContext<Long> ctx) throws Exception { for (Long processingTime : processingTimes) { if (cancelled) { break; } processingTimeService.setCurrentTime(processingTime); } }
@Test public void testCurrentProcessingTime() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); processingTimeService.setCurrentTime(17L); assertEquals(17, timerService.currentProcessingTime()); processingTimeService.setCurrentTime(42); assertEquals(42, timerService.currentProcessingTime()); }
@Test public void testCustomTimeServiceProvider() throws Throwable { final TestProcessingTimeService tp = new TestProcessingTimeService(); tp.setCurrentTime(11); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 11); tp.setCurrentTime(15); tp.setCurrentTime(16); assertEquals(testHarness.getProcessingTimeService().getCurrentProcessingTime(), 16); assertEquals(2, tp.getNumActiveTimers()); tp.setCurrentTime(35); assertEquals(1, tp.getNumActiveTimers()); tp.setCurrentTime(40); assertEquals(0, tp.getNumActiveTimers()); tp.shutdownService();
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L)); processingTimeService.setCurrentTime(10); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L)); processingTimeService.setCurrentTime(20); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L));
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); processingTimeService.setCurrentTime(initialTime); processingTimeService.setCurrentTime(initialTime + idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10); switch (testMethod) { case COLLECT: break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime()); break; case EMIT_WATERMARK: context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime())); break; processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 2 * idleTimeout / 10); switch (testMethod) { case COLLECT: break; case COLLECT_WITH_TIMESTAMP: context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
/** * Verify that registering a processing-time timer that is earlier than the existing timers * removes the one physical timer and creates one for the earlier timestamp * {@link ProcessingTimeService}. */ @Test public void testRegisterEarlierProcessingTimerMovesPhysicalProcessingTimer() throws Exception { @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class); TestKeyContext keyContext = new TestKeyContext(); TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); int key = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism); keyContext.setCurrentKey(key); timerService.registerProcessingTimeTimer("ciao", 20); assertEquals(1, timerService.numProcessingTimeTimers()); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L)); timerService.registerProcessingTimeTimer("ciao", 10); assertEquals(2, timerService.numProcessingTimeTimers()); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L)); }
public long getProcessingTime() { return processingTimeService.getCurrentProcessingTime(); }
processingTimeService = new TestProcessingTimeService(); processingTimeService.setCurrentTime(0);
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); PriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128); assertEquals(3, timerService.numProcessingTimeTimers("ciao")); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(10L)); processingTimeService.setCurrentTime(10); assertEquals(2, timerService.numProcessingTimeTimers("ciao")); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(20L)); processingTimeService.setCurrentTime(20); assertEquals(1, timerService.numProcessingTimeTimers("ciao")); assertEquals(1, processingTimeService.getNumActiveTimers()); assertThat(processingTimeService.getActiveTimerTimestamps(), containsInAnyOrder(30L)); processingTimeService.setCurrentTime(30); assertEquals(0, processingTimeService.getNumActiveTimers()); assertEquals(1, processingTimeService.getNumActiveTimers());
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); processingTimeService.setCurrentTime(initialTime); processingTimeService.setCurrentTime(initialTime + watermarkInterval); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); processingTimeService.setCurrentTime(initialTime + 2 * watermarkInterval); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); processingTimeService.setCurrentTime(initialTime + idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); assertEquals(expectedOutput, output); processingTimeService.setCurrentTime(initialTime + 3 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 4 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout); processingTimeService.setCurrentTime(initialTime + 6 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 7 * watermarkInterval); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle()); assertEquals(expectedOutput, output); processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10); switch (testMethod) { case COLLECT: context.collect("msg"); expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime())); expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval))); assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive()); assertEquals(expectedOutput, output); break;
@Test public void testPeriodicWatermarksWithNoSubscribedPartitionsShouldYieldNoWatermarks() throws Exception { final String testTopic = "test topic name"; Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>(); TestSourceContext<Long> sourceContext = new TestSourceContext<>(); TestProcessingTimeService processingTimeProvider = new TestProcessingTimeService(); TestFetcher<Long> fetcher = new TestFetcher<>( sourceContext, originalPartitions, new SerializedValue<AssignerWithPeriodicWatermarks<Long>>(new PeriodicTestExtractor()), null, /* punctuated watermarks assigner*/ processingTimeProvider, 10); processingTimeProvider.setCurrentTime(10); // no partitions; when the periodic watermark emitter fires, no watermark should be emitted assertFalse(sourceContext.hasWatermark()); // counter-test that when the fetcher does actually have partitions, // when the periodic watermark emitter fires again, a watermark really is emitted fetcher.addDiscoveredPartitions(Collections.singletonList(new KafkaTopicPartition(testTopic, 0))); fetcher.emitRecord(100L, fetcher.subscribedPartitionStates().get(0), 3L); processingTimeProvider.setCurrentTime(20); assertEquals(100, sourceContext.getLatestWatermark().getTimestamp()); }
@Override public boolean shutdownAndAwaitPending(long time, TimeUnit timeUnit) throws InterruptedException { shutdownService(); return true; }
@SafeVarargs private MockFetcher(HashMap<KafkaTopicPartition, Long>... stateSnapshotsToReturn) throws Exception { super( new TestSourceContext<>(), new HashMap<>(), null, null, new TestProcessingTimeService(), 0, MockFetcher.class.getClassLoader(), new UnregisteredMetricsGroup(), false); this.stateSnapshotsToReturn.addAll(Arrays.asList(stateSnapshotsToReturn)); }
public void setProcessingTime(long time) throws Exception { processingTimeService.setCurrentTime(time); }
TestProcessingTimeService processingTimeService = new TestProcessingTimeService(); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, createQueueFactory()); assertEquals(2, timerService.numProcessingTimeTimers("ciao")); processingTimeService.setCurrentTime(10);
/** * Calls close and dispose on the operator. */ public void close() throws Exception { operator.close(); operator.dispose(); if (processingTimeService != null) { processingTimeService.shutdownService(); } setupCalled = false; internalEnvironment.ifPresent(MockEnvironment::close); }
@SuppressWarnings("unchecked") private static <T> void setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, final ProcessingTimeService timeProvider) { ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setAutoWatermarkInterval(watermarkInterval); StreamConfig cfg = new StreamConfig(new Configuration()); cfg.setStateBackend(new MemoryStateBackend()); cfg.setTimeCharacteristic(timeChar); cfg.setOperatorID(new OperatorID()); Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0); StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class); when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE); StreamTask<?, ?> mockTask = mock(StreamTask.class); when(mockTask.getName()).thenReturn("Mock Task"); when(mockTask.getCheckpointLock()).thenReturn(new Object()); when(mockTask.getConfiguration()).thenReturn(cfg); when(mockTask.getEnvironment()).thenReturn(env); when(mockTask.getExecutionConfig()).thenReturn(executionConfig); when(mockTask.getAccumulatorMap()).thenReturn(Collections.<String, Accumulator<?, ?>>emptyMap()); when(mockTask.getStreamStatusMaintainer()).thenReturn(streamStatusMaintainer); doAnswer(new Answer<ProcessingTimeService>() {
private void testLatencyMarkEmission(int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception { final List<StreamElement> output = new ArrayList<>(); final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService(); testProcessingTimeService.setCurrentTime(0L); final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime); // regular stream source operator final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes)); operatorSetup.setupSourceOperator(operator, testProcessingTimeService); // run and wait to be stopped operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output)); assertEquals( numberLatencyMarkers + 1, // + 1 is the final watermark element output.size()); long timestamp = 0L; int i = 0; // verify that its only latency markers + a final watermark for (; i < numberLatencyMarkers; i++) { StreamElement se = output.get(i); Assert.assertTrue(se.isLatencyMarker()); Assert.assertEquals(operator.getOperatorID(), se.asLatencyMarker().getOperatorId()); Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex()); Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp); timestamp += latencyMarkInterval; } Assert.assertTrue(output.get(i).isWatermark()); }