@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { userFunction.timeout(element.getValue(), streamRecordBufferEntry); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete( (StreamElementQueueEntry<Collection<OUT>> value) -> { timerFuture.cancel(true); }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); }
final OrderedStreamElementQueue queue = new OrderedStreamElementQueue(4, executor, operatorActions); StreamRecordQueueEntry<Integer> entry1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); StreamRecordQueueEntry<Integer> entry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); WatermarkQueueEntry entry3 = new WatermarkQueueEntry(new Watermark(2L)); StreamRecordQueueEntry<Integer> entry4 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L)); entry2.complete(Collections.<Integer>emptyList()); entry4.complete(Collections.<Integer>emptyList()); entry1.complete(Collections.<Integer>emptyList());
StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L)); queue.put(watermark1); record2.completeExceptionally(testException); record1.complete(Arrays.asList(1));
@Test public void testPut() throws InterruptedException { OperatorActions operatorActions = mock(OperatorActions.class); StreamElementQueue queue = createStreamElementQueue(2, operatorActions); final Watermark watermark = new Watermark(0L); final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L); final Watermark nextWatermark = new Watermark(2L); final WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(watermark); final StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(streamRecord); queue.put(watermarkQueueEntry); queue.put(streamRecordQueueEntry); Assert.assertEquals(2, queue.size()); Assert.assertFalse(queue.tryPut(new WatermarkQueueEntry(nextWatermark))); Collection<StreamElementQueueEntry<?>> actualValues = queue.values(); List<StreamElementQueueEntry<?>> expectedValues = Arrays.asList(watermarkQueueEntry, streamRecordQueueEntry); Assert.assertEquals(expectedValues, actualValues); verify(operatorActions, never()).failOperator(any(Exception.class)); }
@Override public void onProcessingTime(long timestamp) throws Exception { streamRecordBufferEntry.collect( new TimeoutException("Async function call has timed out.")); } });
@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { userFunction.timeout(element.getValue(), streamRecordBufferEntry); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete( (StreamElementQueueEntry<Collection<OUT>> value) -> { timerFuture.cancel(true); }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); }
final StreamElementQueue queue = createStreamElementQueue(1, operatorActions); StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 0L)); final StreamRecordQueueEntry<Integer> streamRecordQueueEntry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(43, 1L)); streamRecordQueueEntry.complete(Collections.<Integer>emptyList());
@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { userFunction.timeout(element.getValue(), streamRecordBufferEntry); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete( (StreamElementQueueEntry<Collection<OUT>> value) -> { timerFuture.cancel(true); }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); }
StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L)); StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 4L)); queue.put(record3); record2.complete(Arrays.asList(3, 4)); record1.complete(Arrays.asList(1, 2)); record3.complete(Arrays.asList(5, 6));
@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { streamRecordBufferEntry.collect( new TimeoutException("Async function call has timed out.")); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete(new AcceptFunction<StreamElementQueueEntry<Collection<OUT>>>() { @Override public void accept(StreamElementQueueEntry<Collection<OUT>> value) { timerFuture.cancel(true); } }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); }
StreamRecordQueueEntry<Integer> record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); StreamRecordQueueEntry<Integer> record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(2L)); StreamRecordQueueEntry<Integer> record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L)); StreamRecordQueueEntry<Integer> record4 = new StreamRecordQueueEntry<>(new StreamRecord<>(4, 4L)); WatermarkQueueEntry watermark2 = new WatermarkQueueEntry(new Watermark(5L)); StreamRecordQueueEntry<Integer> record5 = new StreamRecordQueueEntry<>(new StreamRecord<>(5, 6L)); StreamRecordQueueEntry<Integer> record6 = new StreamRecordQueueEntry<>(new StreamRecord<>(6, 7L)); record3.complete(Collections.<Integer>emptyList()); record2.complete(Collections.<Integer>emptyList()); executor); record6.complete(Collections.<Integer>emptyList()); record4.complete(Collections.<Integer>emptyList()); record1.complete(Collections.<Integer>emptyList()); record5.complete(Collections.<Integer>emptyList());
StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 2L)); streamRecordQueueEntry.complete(Collections.<Integer>emptyList());
@Test public void testPoll() throws InterruptedException { OperatorActions operatorActions = mock(OperatorActions.class); StreamElementQueue queue = createStreamElementQueue(2, operatorActions); WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(0L)); StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 1L)); queue.put(watermarkQueueEntry); queue.put(streamRecordQueueEntry); Assert.assertEquals(watermarkQueueEntry, queue.peekBlockingly()); Assert.assertEquals(2, queue.size()); Assert.assertEquals(watermarkQueueEntry, queue.poll()); Assert.assertEquals(1, queue.size()); streamRecordQueueEntry.complete(Collections.<Integer>emptyList()); Assert.assertEquals(streamRecordQueueEntry, queue.poll()); Assert.assertEquals(0, queue.size()); Assert.assertTrue(queue.isEmpty()); verify(operatorActions, never()).failOperator(any(Exception.class)); }