@Override public int compare(ProducerBatch o1, ProducerBatch o2) { return o1.baseSequence() - o2.baseSequence(); } }));
public boolean hasSequence() { return baseSequence() != RecordBatch.NO_SEQUENCE; }
/** * Returns the first inflight sequence for a given partition. This is the base sequence of an inflight batch with * the lowest sequence number. * @return the lowest inflight sequence if the transaction manager is tracking inflight requests for this partition. * If there are no inflight requests being tracked for this partition, this method will return * RecordBatch.NO_SEQUENCE. */ synchronized int firstInFlightSequence(TopicPartition topicPartition) { PriorityQueue<ProducerBatch> inFlightBatches = inflightBatchesBySequence.get(topicPartition); if (inFlightBatches == null) return RecordBatch.NO_SEQUENCE; ProducerBatch firstInFlightBatch = inFlightBatches.peek(); if (firstInFlightBatch == null) return RecordBatch.NO_SEQUENCE; return firstInFlightBatch.baseSequence(); }
private void insertInSequenceOrder(Deque<ProducerBatch> deque, ProducerBatch batch) { if (batch.baseSequence() == RecordBatch.NO_SEQUENCE) throw new IllegalStateException("Trying to re-enqueue a batch which doesn't have a sequence even " + "though idempotency is enabled."); "requests. batch.topicPartition: " + batch.topicPartition + "; batch.baseSequence: " + batch.baseSequence()); if (firstBatchInQueue != null && firstBatchInQueue.hasSequence() && firstBatchInQueue.baseSequence() < batch.baseSequence()) { while (deque.peekFirst() != null && deque.peekFirst().hasSequence() && deque.peekFirst().baseSequence() < batch.baseSequence()) orderedBatches.add(deque.pollFirst()); "position {}", batch.baseSequence(), batch.topicPartition, orderedBatches.size());
synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { if (!this.nextSequence.containsKey(batch.topicPartition)) // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; log.debug("producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", batch.producerId(), batch.topicPartition, batch.recordCount); int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; if (currentSequence < 0) throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative : " + currentSequence); setNextSequence(batch.topicPartition, currentSequence); for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(batch.topicPartition)) { if (inFlightBatch.baseSequence() < batch.baseSequence()) continue; int newSequence = inFlightBatch.baseSequence() - batch.recordCount; if (newSequence < 0) throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), batch.topicPartition, newSequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); } }
private synchronized void startSequencesAtBeginning(TopicPartition topicPartition) { int sequence = 0; for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(topicPartition)) { log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), inFlightBatch.topicPartition, sequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), sequence, inFlightBatch.isTransactional()); sequence += inFlightBatch.recordCount; } setNextSequence(topicPartition, sequence); lastAckedSequence.remove(topicPartition); }
int sequence = baseSequence(); ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch()); for (ProducerBatch newBatch : batches) {
log.debug("Assigned producerId {} and producerEpoch {} to batch with base sequence " + "{} being sent to partition {}", producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, batch.baseSequence(), tp);
private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) { if (transactionManager != null) { if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { transactionManager .maybeUpdateLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, transactionManager.lastAckedSequence(batch.topicPartition)); } transactionManager.updateLastAckedOffset(response, batch); transactionManager.removeInFlightBatch(batch); } if (batch.done(response.baseOffset, response.logAppendTime, null)) { maybeRemoveFromInflightBatches(batch); this.accumulator.deallocate(batch); } }
private boolean shouldStopDrainBatchesForPartition(ProducerBatch first, TopicPartition tp) { ProducerIdAndEpoch producerIdAndEpoch = null; if (transactionManager != null) { if (!transactionManager.isSendToPartitionAllowed(tp)) return true; producerIdAndEpoch = transactionManager.producerIdAndEpoch(); if (!producerIdAndEpoch.isValid()) // we cannot send the batch until we have refreshed the producer id return true; if (!first.hasSequence() && transactionManager.hasUnresolvedSequence(first.topicPartition)) // Don't drain any new batches while the state of previous sequence numbers // is unknown. The previous batches would be unknown if they were aborted // on the client after being sent to the broker at least once. return true; int firstInFlightSequence = transactionManager.firstInFlightSequence(first.topicPartition); if (firstInFlightSequence != RecordBatch.NO_SEQUENCE && first.hasSequence() && first.baseSequence() != firstInFlightSequence) // If the queued batch already has an assigned sequence, then it is being retried. // In this case, we wait until the next immediate batch is ready and drain that. // We only move on when the next in line batch is complete (either successfully or due to // a fatal broker error). This effectively reduces our in flight request count to 1. return true; } return false; }
(batch.sequenceHasBeenReset() || !isNextSequence(batch.topicPartition, batch.baseSequence())))
batch.topicPartition, batch.producerId(), batch.baseSequence()); reenqueueBatch(batch, now); } else {
assertEquals(1, queuedBatches.peekFirst().baseSequence()); assertEquals(1, client.inFlightRequestCount()); assertEquals(-1, transactionManager.lastAckedSequence(tp0)); assertEquals(0, queuedBatches.peekFirst().baseSequence()); assertEquals(1, queuedBatches.peekLast().baseSequence()); assertEquals(-1, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount());
assertEquals(0, queuedBatches.peekFirst().baseSequence()); assertEquals(1, transactionManager.lastAckedSequence(tp0)); assertEquals(0, client.inFlightRequestCount());