public StreamingSubscriberConnection( String subscription, MessageReceiver receiver, Duration ackExpirationPadding, Duration maxAckExtensionPeriod, Distribution ackLatencyDistribution, SubscriberStub stub, int channelAffinity, FlowController flowController, Deque<MessageDispatcher.OutstandingMessageBatch> outstandingMessageBatches, ScheduledExecutorService executor, ScheduledExecutorService systemExecutor, ApiClock clock) { this.subscription = subscription; this.systemExecutor = systemExecutor; this.stub = stub; this.channelAffinity = channelAffinity; this.messageDispatcher = new MessageDispatcher( receiver, this, ackExpirationPadding, maxAckExtensionPeriod, ackLatencyDistribution, flowController, outstandingMessageBatches, executor, systemExecutor, clock); }
@Override public void run() { try { if (extendDeadline.getAndSet(false)) { int newDeadlineSec = computeDeadlineSeconds(); messageDeadlineSeconds.set(newDeadlineSec); extendDeadlines(); // Don't bother cancelling this when we stop. It'd just set an atomic boolean. systemExecutor.schedule( setExtendDeadline, newDeadlineSec - ackExpirationPadding.getSeconds(), TimeUnit.SECONDS); } processOutstandingAckOperations(); } catch (Throwable t) { // Catch everything so that one run failing doesn't prevent subsequent runs. logger.log(Level.WARNING, "failed to run periodic job", t); } } },
@Override protected void doStop() { messageDispatcher.stop(); lock.lock(); try { clientStream.closeSendWithError(Status.CANCELLED.asException()); } finally { lock.unlock(); notifyStopped(); } }
@Test public void testAck() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); consumers.take().ack(); dispatcher.processOutstandingAckOperations(); assertThat(sentAcks).contains(TEST_MESSAGE.getAckId()); }
@Test public void testExtension_GiveUp() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); dispatcher.extendDeadlines(); assertThat(sentModAcks) .contains(ModAckItem.of(TEST_MESSAGE.getAckId(), Subscriber.MIN_ACK_DEADLINE_SECONDS)); sentModAcks.clear(); // If we run extendDeadlines after totalExpiration, we shouldn't send anything. // In particular, don't send negative modacks. clock.advance(1, TimeUnit.DAYS); dispatcher.extendDeadlines(); assertThat(sentModAcks).isEmpty(); // We should be able to reserve another item in the flow controller and not block shutdown flowController.reserve(1, 0); dispatcher.stop(); }
@Test public void testExtension() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); dispatcher.extendDeadlines(); assertThat(sentModAcks) .contains(ModAckItem.of(TEST_MESSAGE.getAckId(), Subscriber.MIN_ACK_DEADLINE_SECONDS)); sentModAcks.clear(); consumers.take().ack(); dispatcher.extendDeadlines(); assertThat(sentModAcks).isEmpty(); }
@Test public void testDeadlineAdjustment() throws Exception { assertThat(dispatcher.computeDeadlineSeconds()).isEqualTo(10); dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); clock.advance(42, TimeUnit.SECONDS); consumers.take().ack(); assertThat(dispatcher.computeDeadlineSeconds()).isEqualTo(42); } }
new MessageDispatcher( receiver, processor, systemExecutor, clock); dispatcher.setMessageDeadlineSeconds(Subscriber.MIN_ACK_DEADLINE_SECONDS);
public void stop() { messagesWaiter.waitNoMessages(); jobLock.lock(); try { if (backgroundJob != null) { backgroundJob.cancel(false); backgroundJob = null; } } finally { jobLock.unlock(); } processOutstandingAckOperations(); }
@Override public void onResponse(StreamingPullResponse response) { channelReconnectBackoffMillis.set(INITIAL_CHANNEL_RECONNECT_BACKOFF.toMillis()); messageDispatcher.processReceivedMessages( response.getReceivedMessagesList(), new Runnable() { @Override public void run() { // Only request more if we're not shutdown. // If errorFuture is done, the stream has either failed or hung up, // and we don't need to request. if (isAlive() && !errorFuture.isDone()) { lock.lock(); try { thisController.request(1); } catch (Exception e) { logger.log(Level.WARNING, "cannot request more messages", e); } finally { lock.unlock(); } } } }); }
@InternalApi void extendDeadlines() { int extendSeconds = getMessageDeadlineSeconds(); List<PendingModifyAckDeadline> modacks = new ArrayList<>(); PendingModifyAckDeadline modack = new PendingModifyAckDeadline(extendSeconds); Instant now = now(); Instant extendTo = now.plusSeconds(extendSeconds); for (Map.Entry<String, AckHandler> entry : pendingMessages.entrySet()) { String ackId = entry.getKey(); Instant totalExpiration = entry.getValue().totalExpiration; if (totalExpiration.isAfter(extendTo)) { modack.ackIds.add(ackId); continue; } // forget removes from pendingMessages; this is OK, concurrent maps can // handle concurrent iterations and modifications. entry.getValue().forget(); if (totalExpiration.isAfter(now)) { int sec = Math.max(1, (int) now.until(totalExpiration, ChronoUnit.SECONDS)); modacks.add(new PendingModifyAckDeadline(sec, ackId)); } } logger.log(Level.FINER, "Sending {0} modacks", modack.ackIds.size() + modacks.size()); modacks.add(modack); List<String> acksToSend = Collections.emptyList(); ackProcessor.sendAckOperations(acksToSend, modacks); }
Instant totalExpiration = now().plus(maxAckExtensionPeriod); OutstandingMessageBatch outstandingBatch = new OutstandingMessageBatch(doneCallback); for (ReceivedMessage message : messages) { outstandingMessageBatches.add(outstandingBatch); processOutstandingBatches();
/** Stop extending deadlines for this message and free flow control. */ private void forget() { if (pendingMessages.remove(ackId) == null) { /* * We're forgetting the message for the second time. Probably because we ran out of total * expiration, forget the message, then the user finishes working on the message, and forget * again. Turn the second forget into a no-op so we don't free twice. */ return; } flowController.release(1, outstandingBytes); messagesWaiter.incrementPendingMessages(-1); processOutstandingBatches(); }
@InternalApi void processOutstandingAckOperations() { List<PendingModifyAckDeadline> modifyAckDeadlinesToSend = new ArrayList<>(); List<String> acksToSend = new ArrayList<>(); pendingAcks.drainTo(acksToSend); logger.log(Level.FINER, "Sending {0} acks", acksToSend.size()); PendingModifyAckDeadline nacksToSend = new PendingModifyAckDeadline(0); pendingNacks.drainTo(nacksToSend.ackIds); logger.log(Level.FINER, "Sending {0} nacks", nacksToSend.ackIds.size()); if (!nacksToSend.ackIds.isEmpty()) { modifyAckDeadlinesToSend.add(nacksToSend); } PendingModifyAckDeadline receiptsToSend = new PendingModifyAckDeadline(getMessageDeadlineSeconds()); pendingReceipts.drainTo(receiptsToSend.ackIds); logger.log(Level.FINER, "Sending {0} receipts", receiptsToSend.ackIds.size()); if (!receiptsToSend.ackIds.isEmpty()) { modifyAckDeadlinesToSend.add(receiptsToSend); } ackProcessor.sendAckOperations(acksToSend, modifyAckDeadlinesToSend); }
@Test public void testReceipt() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); dispatcher.processOutstandingAckOperations(); assertThat(sentModAcks) .contains(ModAckItem.of(TEST_MESSAGE.getAckId(), Subscriber.MIN_ACK_DEADLINE_SECONDS)); }
@Test public void testExtension_Close() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); dispatcher.extendDeadlines(); assertThat(sentModAcks) .contains(ModAckItem.of(TEST_MESSAGE.getAckId(), Subscriber.MIN_ACK_DEADLINE_SECONDS)); sentModAcks.clear(); // Default total expiration is an hour (60*60 seconds). We normally would extend by 10s. // However, only extend by 5s here, since there's only 5s left before total expiration. clock.advance(60 * 60 - 5, TimeUnit.SECONDS); dispatcher.extendDeadlines(); assertThat(sentModAcks).contains(ModAckItem.of(TEST_MESSAGE.getAckId(), 5)); }
@Test public void testNack() throws Exception { dispatcher.processReceivedMessages(Collections.singletonList(TEST_MESSAGE), NOOP_RUNNABLE); consumers.take().nack(); dispatcher.processOutstandingAckOperations(); assertThat(sentModAcks).contains(ModAckItem.of(TEST_MESSAGE.getAckId(), 0)); }