/** * Mark all partitions as ready to send and block until the send is complete */ public void awaitFlushCompletion() throws InterruptedException { try { for (ProducerBatch batch : this.incomplete.copyAll()) batch.produceFuture.await(); } finally { this.flushesInProgress.decrementAndGet(); } }
/** * Check whether there are any pending batches (whether sent or unsent). */ public boolean hasIncomplete() { return !this.incomplete.isEmpty(); }
/** * Split the big batch that has been rejected and reenqueue the split batches in to the accumulator. * @return the number of split batches. */ public int splitAndReenqueue(ProducerBatch bigBatch) { // Reset the estimated compression ratio to the initial value or the big batch compression ratio, whichever // is bigger. There are several different ways to do the reset. We chose the most conservative one to ensure // the split doesn't happen too often. CompressionRatioEstimator.setEstimation(bigBatch.topicPartition.topic(), compression, Math.max(1.0f, (float) bigBatch.compressionRatio())); Deque<ProducerBatch> dq = bigBatch.split(this.batchSize); int numSplitBatches = dq.size(); Deque<ProducerBatch> partitionDequeue = getOrCreateDeque(bigBatch.topicPartition); while (!dq.isEmpty()) { ProducerBatch batch = dq.pollLast(); incomplete.add(batch); // We treat the newly split batches as if they are not even tried. synchronized (partitionDequeue) { if (transactionManager != null) { // We should track the newly created batches since they already have assigned sequences. transactionManager.addInFlightBatch(batch); insertInSequenceOrder(partitionDequeue, batch); } else { partitionDequeue.addFirst(batch); } } } return numSplitBatches; }
this.batches = new CopyOnWriteMap<>(); this.free = bufferPool; this.incomplete = new IncompleteBatches(); this.muted = new HashMap<>(); this.time = time;
/** * Deallocate the record batch */ public void deallocate(ProducerBatch batch) { incomplete.remove(batch); // Only deallocate the batch if it is not a split batch because split batch are allocated outside the // buffer pool. if (!batch.isSplitBatch()) free.deallocate(batch.buffer(), batch.initialCapacity()); }
incomplete.add(batch);
/** * Abort all incomplete batches (whether they have been sent or not) */ void abortBatches(final RuntimeException reason) { for (ProducerBatch batch : incomplete.copyAll()) { Deque<ProducerBatch> dq = getDeque(batch.topicPartition); synchronized (dq) { batch.abortRecordAppends(); dq.remove(batch); } batch.abort(reason); deallocate(batch); } }
/** * Abort any batches which have not been drained */ void abortUndrainedBatches(RuntimeException reason) { for (ProducerBatch batch : incomplete.copyAll()) { Deque<ProducerBatch> dq = getDeque(batch.topicPartition); boolean aborted = false; synchronized (dq) { if ((transactionManager != null && !batch.hasSequence()) || (transactionManager == null && !batch.isClosed())) { aborted = true; batch.abortRecordAppends(); dq.remove(batch); } } if (aborted) { batch.abort(reason); deallocate(batch); } } }