private boolean shouldStopDrainBatchesForPartition(ProducerBatch first, TopicPartition tp) { ProducerIdAndEpoch producerIdAndEpoch = null; if (transactionManager != null) { if (!transactionManager.isSendToPartitionAllowed(tp)) return true; producerIdAndEpoch = transactionManager.producerIdAndEpoch(); if (!producerIdAndEpoch.isValid()) // we cannot send the batch until we have refreshed the producer id return true; if (!first.hasSequence() && transactionManager.hasUnresolvedSequence(first.topicPartition)) // Don't drain any new batches while the state of previous sequence numbers // is unknown. The previous batches would be unknown if they were aborted // on the client after being sent to the broker at least once. return true; int firstInFlightSequence = transactionManager.firstInFlightSequence(first.topicPartition); if (firstInFlightSequence != RecordBatch.NO_SEQUENCE && first.hasSequence() && first.baseSequence() != firstInFlightSequence) // If the queued batch already has an assigned sequence, then it is being retried. // In this case, we wait until the next immediate batch is ready and drain that. // We only move on when the next in line batch is complete (either successfully or due to // a fatal broker error). This effectively reduces our in flight request count to 1. return true; } return false; }
if (error == Errors.OUT_OF_ORDER_SEQUENCE_NUMBER && !hasUnresolvedSequence(batch.topicPartition) && (batch.sequenceHasBeenReset() || !isNextSequence(batch.topicPartition, batch.baseSequence())))
@Test public void testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences() throws Exception { final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); setupWithTransactionState(transactionManager); prepareAndReceiveInitProducerId(producerId, Errors.NONE); assertTrue(transactionManager.hasProducerId()); assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); // Send first ProduceRequest Future<RecordMetadata> request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; Node node = metadata.fetch().nodes().get(0); time.sleep(10000L); client.disconnect(node.idString()); client.blackout(node, 10); sender.run(time.milliseconds()); assertFutureFailure(request1, TimeoutException.class); assertFalse(transactionManager.hasUnresolvedSequence(tp0)); }
assertTrue(transactionManager.hasUnresolvedSequence(tp0)); assertEquals(0, sender.inFlightBatches(tp0).size()); assertFalse(client.hasInFlightRequests()); assertEquals(2L, transactionManager.sequenceNumber(tp0).longValue()); assertTrue(transactionManager.hasUnresolvedSequence(tp0)); assertFalse(transactionManager.hasUnresolvedSequence(tp0)); assertTrue(transactionManager.hasProducerId()); assertEquals(0, batches.size());
assertTrue(transactionManager.hasUnresolvedSequence(tp0)); assertFalse(transactionManager.hasUnresolvedSequence(tp0));
@Test public void testExpiryOfAllSentBatchesShouldCauseUnresolvedSequences() throws Exception { final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); setupWithTransactionState(transactionManager); prepareAndReceiveInitProducerId(producerId, Errors.NONE); assertTrue(transactionManager.hasProducerId()); assertEquals(0, transactionManager.sequenceNumber(tp0).longValue()); // Send first ProduceRequest Future<RecordMetadata> request1 = accumulator.append(tp0, 0L, "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); // send request sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_FOR_PARTITION, -1); sender.run(time.milliseconds()); // receive response assertEquals(1L, transactionManager.sequenceNumber(tp0).longValue()); Node node = metadata.fetch().nodes().get(0); time.sleep(15000L); client.disconnect(node.idString()); client.blackout(node, 10); sender.run(time.milliseconds()); // now expire the batch. assertFutureFailure(request1, TimeoutException.class); assertTrue(transactionManager.hasUnresolvedSequence(tp0)); assertFalse(client.hasInFlightRequests()); Deque<ProducerBatch> batches = accumulator.batches().get(tp0); assertEquals(0, batches.size()); assertTrue(transactionManager.hasProducerId(producerId)); // We should now clear the old producerId and get a new one in a single run loop. prepareAndReceiveInitProducerId(producerId + 1, Errors.NONE); assertTrue(transactionManager.hasProducerId(producerId + 1)); }