@Override public void processWatermark(Watermark mark) throws Exception { WatermarkQueueEntry watermarkBufferEntry = new WatermarkQueueEntry(mark); addAsyncBufferEntry(watermarkBufferEntry); }
@Override public void run() { try { while (running) { LOG.debug("Wait for next completed async stream element result."); AsyncResult streamElementEntry = streamElementQueue.peekBlockingly(); output(streamElementEntry); } } catch (InterruptedException e) { if (running) { operatorActions.failOperator(e); } else { // Thread got interrupted which means that it should shut down LOG.debug("Emitter thread got interrupted, shutting down."); } } catch (Throwable t) { operatorActions.failOperator(new Exception("AsyncWaitOperator's emitter caught an " + "unexpected throwable.", t)); } }
@Override public void failOperator(Throwable throwable) { getContainingTask().getEnvironment().failExternally(throwable); } }
private void testProcessingTime(AsyncDataStream.OutputMode mode) throws Exception { final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>( new MyAsyncFunction(), TIMEOUT, 6, mode); expectedOutput, testHarness.getOutput(), new StreamRecordComparator());
private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>(); long timeout = 10L; // 1 milli second AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncFunction, timeout, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } harness.setProcessingTime(10L); synchronized (harness.getCheckpointLock()) { harness.close(); } }
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction(); long timeout = 2000L; AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncWaitFunction, TIMEOUT, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } synchronized (harness.getCheckpointLock()) { harness.close(); } assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent()); }
@Override public void dispose() throws Exception { Exception exception = null; try { super.dispose(); } catch (InterruptedException interrupted) { exception = interrupted; Thread.currentThread().interrupt(); } catch (Exception e) { exception = e; } try { stopResources(false); } catch (InterruptedException interrupted) { exception = ExceptionUtils.firstOrSuppressed(interrupted, exception); Thread.currentThread().interrupt(); } catch (Exception e) { exception = ExceptionUtils.firstOrSuppressed(e, exception); } if (exception != null) { throw exception; } }
@Test public void testAsyncTimeoutFailure() throws Exception { testAsyncTimeout( new LazyAsyncFunction(), Optional.of(TimeoutException.class), new StreamRecord<>(2, 5L)); }
@Test public void testAsyncTimeoutIgnore() throws Exception { testAsyncTimeout( new IgnoreTimeoutLazyAsyncFunction(), Optional.empty(), new StreamRecord<>(3, 0L), new StreamRecord<>(2, 5L)); }
/** * Test the AsyncWaitOperator with unordered mode and processing time. */ @Test public void testProcessingUnordered() throws Exception { testProcessingTime(AsyncDataStream.OutputMode.UNORDERED); }
/** * FLINK-6435 * * <p>Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until * another StreamElementQueueEntry is properly completed before it is collected. */ @Test(timeout = 2000) public void testOrderedWaitUserExceptionHandling() throws Exception { testUserExceptionHandling(AsyncDataStream.OutputMode.ORDERED); }
/** * Test the AsyncWaitOperator with ordered mode and event time. */ @Test public void testEventTimeOrdered() throws Exception { testEventTime(AsyncDataStream.OutputMode.ORDERED); }
/** * FLINK-6435 * * <p>Tests that timeout exceptions are properly handled in ordered output mode. The proper handling means that * a StreamElementQueueEntry is completed in case of a timeout exception. */ @Test public void testUnorderedWaitTimeoutHandling() throws Exception { testTimeoutExceptionHandling(AsyncDataStream.OutputMode.UNORDERED); }
@Override public void close() throws Exception { super.close(); freeExecutor(); }
@Override public void close() throws Exception { closingLatch.trigger(); checkpointingLock.notifyAll(); super.close(); } }
private void testEventTime(AsyncDataStream.OutputMode mode) throws Exception { final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>( new MyAsyncFunction(), TIMEOUT, 2, expectedOutput, testHarness.getOutput(), new StreamRecordComparator());
/** * Test the AsyncWaitOperator with ordered mode and processing time. */ @Test public void testProcessingTimeOrdered() throws Exception { testProcessingTime(AsyncDataStream.OutputMode.ORDERED); }
/** * FLINK-6435 * * <p>Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until * another StreamElementQueueEntry is properly completed before it is collected. */ @Test(timeout = 2000) public void testUnorderedWaitUserExceptionHandling() throws Exception { testUserExceptionHandling(AsyncDataStream.OutputMode.UNORDERED); }
/** * Test the AsyncWaitOperator with unordered mode and event time. */ @Test public void testWaterMarkUnordered() throws Exception { testEventTime(AsyncDataStream.OutputMode.UNORDERED); }
/** * FLINK-6435 * * <p>Tests that timeout exceptions are properly handled in ordered output mode. The proper handling means that * a StreamElementQueueEntry is completed in case of a timeout exception. */ @Test public void testOrderedWaitTimeoutHandling() throws Exception { testTimeoutExceptionHandling(AsyncDataStream.OutputMode.ORDERED); }