@Test public void testPoll() throws InterruptedException { OperatorActions operatorActions = mock(OperatorActions.class); StreamElementQueue queue = createStreamElementQueue(2, operatorActions); WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(0L)); StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 1L)); queue.put(watermarkQueueEntry); queue.put(streamRecordQueueEntry); Assert.assertEquals(watermarkQueueEntry, queue.peekBlockingly()); Assert.assertEquals(2, queue.size()); Assert.assertEquals(watermarkQueueEntry, queue.poll()); Assert.assertEquals(1, queue.size()); streamRecordQueueEntry.complete(Collections.<Integer>emptyList()); Assert.assertEquals(streamRecordQueueEntry, queue.poll()); Assert.assertEquals(0, queue.size()); Assert.assertTrue(queue.isEmpty()); verify(operatorActions, never()).failOperator(any(Exception.class)); }
@Test public void testPut() throws InterruptedException { OperatorActions operatorActions = mock(OperatorActions.class); StreamElementQueue queue = createStreamElementQueue(2, operatorActions); final Watermark watermark = new Watermark(0L); final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L); final Watermark nextWatermark = new Watermark(2L); final WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(watermark); final StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(streamRecord); queue.put(watermarkQueueEntry); queue.put(streamRecordQueueEntry); Assert.assertEquals(2, queue.size()); Assert.assertFalse(queue.tryPut(new WatermarkQueueEntry(nextWatermark))); Collection<StreamElementQueueEntry<?>> actualValues = queue.values(); List<StreamElementQueueEntry<?>> expectedValues = Arrays.asList(watermarkQueueEntry, streamRecordQueueEntry); Assert.assertEquals(expectedValues, actualValues); verify(operatorActions, never()).failOperator(any(Exception.class)); }
final StreamRecordQueueEntry<Integer> streamRecordQueueEntry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(43, 1L)); queue.put(streamRecordQueueEntry); Assert.assertEquals(1, queue.size()); queue.put(streamRecordQueueEntry2); } catch (InterruptedException e) { throw new CompletionException(e); Assert.assertEquals(streamRecordQueueEntry, queue.poll());
@Override public void run() { try { while (running) { LOG.debug("Wait for next completed async stream element result."); AsyncResult streamElementEntry = streamElementQueue.peekBlockingly(); output(streamElementEntry); } } catch (InterruptedException e) { if (running) { operatorActions.failOperator(e); } else { // Thread got interrupted which means that it should shut down LOG.debug("Emitter thread got interrupted, shutting down."); } } catch (Throwable t) { operatorActions.failOperator(new Exception("AsyncWaitOperator's emitter caught an " + "unexpected throwable.", t)); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); ListState<StreamElement> partitionableState = getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer)); partitionableState.clear(); Collection<StreamElementQueueEntry<?>> values = queue.values(); try { for (StreamElementQueueEntry<?> value : values) { partitionableState.add(value.getStreamElement()); } // add the pending stream element queue entry if the stream element queue is currently full if (pendingStreamElementQueueEntry != null) { partitionableState.add(pendingStreamElementQueueEntry.getStreamElement()); } } catch (Exception e) { partitionableState.clear(); throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e); } }
/** * Add the given stream element queue entry to the operator's stream element queue. This * operation blocks until the element has been added. * * <p>For that it tries to put the element into the queue and if not successful then it waits on * the checkpointing lock. The checkpointing lock is also used by the {@link Emitter} to output * elements. The emitter is also responsible for notifying this method if the queue has capacity * left again, by calling notifyAll on the checkpointing lock. * * @param streamElementQueueEntry to add to the operator's queue * @param <T> Type of the stream element queue entry's result * @throws InterruptedException if the current thread has been interrupted */ private <T> void addAsyncBufferEntry(StreamElementQueueEntry<T> streamElementQueueEntry) throws InterruptedException { assert(Thread.holdsLock(checkpointingLock)); pendingStreamElementQueueEntry = streamElementQueueEntry; while (!queue.tryPut(streamElementQueueEntry)) { // we wait for the emitter to notify us if the queue has space left again checkpointingLock.wait(); } pendingStreamElementQueueEntry = null; }
assert(Thread.holdsLock(checkpointingLock)); while (!queue.isEmpty()) {
@Override public void run() { try { while (running) { LOG.debug("Wait for next completed async stream element result."); AsyncResult streamElementEntry = streamElementQueue.peekBlockingly(); output(streamElementEntry); } } catch (InterruptedException e) { if (running) { operatorActions.failOperator(e); } else { // Thread got interrupted which means that it should shut down LOG.debug("Emitter thread got interrupted, shutting down."); } } catch (Throwable t) { operatorActions.failOperator(new Exception("AsyncWaitOperator's emitter caught an " + "unexpected throwable.", t)); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); ListState<StreamElement> partitionableState = getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer)); partitionableState.clear(); Collection<StreamElementQueueEntry<?>> values = queue.values(); try { for (StreamElementQueueEntry<?> value : values) { partitionableState.add(value.getStreamElement()); } // add the pending stream element queue entry if the stream element queue is currently full if (pendingStreamElementQueueEntry != null) { partitionableState.add(pendingStreamElementQueueEntry.getStreamElement()); } } catch (Exception e) { partitionableState.clear(); throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e); } }
/** * Add the given stream element queue entry to the operator's stream element queue. This * operation blocks until the element has been added. * * <p>For that it tries to put the element into the queue and if not successful then it waits on * the checkpointing lock. The checkpointing lock is also used by the {@link Emitter} to output * elements. The emitter is also responsible for notifying this method if the queue has capacity * left again, by calling notifyAll on the checkpointing lock. * * @param streamElementQueueEntry to add to the operator's queue * @param <T> Type of the stream element queue entry's result * @throws InterruptedException if the current thread has been interrupted */ private <T> void addAsyncBufferEntry(StreamElementQueueEntry<T> streamElementQueueEntry) throws InterruptedException { assert(Thread.holdsLock(checkpointingLock)); pendingStreamElementQueueEntry = streamElementQueueEntry; while (!queue.tryPut(streamElementQueueEntry)) { // we wait for the emitter to notify us if the queue has space left again checkpointingLock.wait(); } pendingStreamElementQueueEntry = null; }
assert(Thread.holdsLock(checkpointingLock)); while (!queue.isEmpty()) {
StreamRecordQueueEntry<Integer> streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 2L)); Assert.assertTrue(queue.isEmpty()); return queue.peekBlockingly(); } catch (InterruptedException e) { throw new CompletionException(e); queue.put(watermarkQueueEntry); Assert.assertEquals(1, queue.size()); Assert.assertEquals(watermarkQueueEntry, queue.poll()); Assert.assertTrue(queue.isEmpty()); return queue.poll(); } catch (InterruptedException e) { throw new CompletionException(e); queue.put(streamRecordQueueEntry); Assert.assertTrue(queue.isEmpty());
@Override public void run() { try { while (running) { LOG.debug("Wait for next completed async stream element result."); AsyncResult streamElementEntry = streamElementQueue.peekBlockingly(); output(streamElementEntry); } } catch (InterruptedException e) { if (running) { operatorActions.failOperator(e); } else { // Thread got interrupted which means that it should shut down LOG.debug("Emitter thread got interrupted, shutting down."); } } catch (Throwable t) { operatorActions.failOperator(new Exception("AsyncWaitOperator's emitter caught an " + "unexpected throwable.", t)); } }
@Override public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); ListState<StreamElement> partitionableState = getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer)); partitionableState.clear(); Collection<StreamElementQueueEntry<?>> values = queue.values(); try { for (StreamElementQueueEntry<?> value : values) { partitionableState.add(value.getStreamElement()); } // add the pending stream element queue entry if the stream element queue is currently full if (pendingStreamElementQueueEntry != null) { partitionableState.add(pendingStreamElementQueueEntry.getStreamElement()); } } catch (Exception e) { partitionableState.clear(); throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e); } }
/** * Add the given stream element queue entry to the operator's stream element queue. This * operation blocks until the element has been added. * * <p>For that it tries to put the element into the queue and if not successful then it waits on * the checkpointing lock. The checkpointing lock is also used by the {@link Emitter} to output * elements. The emitter is also responsible for notifying this method if the queue has capacity * left again, by calling notifyAll on the checkpointing lock. * * @param streamElementQueueEntry to add to the operator's queue * @param <T> Type of the stream element queue entry's result * @throws InterruptedException if the current thread has been interrupted */ private <T> void addAsyncBufferEntry(StreamElementQueueEntry<T> streamElementQueueEntry) throws InterruptedException { assert(Thread.holdsLock(checkpointingLock)); pendingStreamElementQueueEntry = streamElementQueueEntry; while (!queue.tryPut(streamElementQueueEntry)) { // we wait for the emitter to notify us if the queue has space left again checkpointingLock.wait(); } pendingStreamElementQueueEntry = null; }
assert(Thread.holdsLock(checkpointingLock)); while (!queue.isEmpty()) {