private void doBatchSendAndAdd(MessageImpl msg, SendCallback callback, ByteBuf payload) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Closing out batch to accomodate large message with size {}", topic, producerName, msg.getDataBuffer().readableBytes()); } batchMessageAndSend(); batchMessageContainer.add(msg, callback); payload.release(); }
/** * fail any pending batch messages that were enqueued, however batch was not closed out * */ private void failPendingBatchMessages(PulsarClientException ex) { if (batchMessageContainer.isEmpty()) { return; } int numMessagesInBatch = batchMessageContainer.numMessagesInBatch; semaphore.release(numMessagesInBatch); try { // Need to protect ourselves from any exception being thrown in the future handler from the application batchMessageContainer.firstCallback.sendComplete(ex); } catch (Throwable t) { log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topic, producerName, batchMessageContainer.sequenceId, t); } ReferenceCountUtil.safeRelease(batchMessageContainer.getBatchedSingleMessageMetadataAndPayload()); batchMessageContainer.clear(); }
int numMessagesInBatch = 0; try { if (!batchMessageContainer.isEmpty()) { numMessagesInBatch = batchMessageContainer.numMessagesInBatch; ByteBuf compressedPayload = batchMessageContainer.getCompressedBatchMetadataAndPayload(); long sequenceId = batchMessageContainer.sequenceId; ByteBuf cmd = sendMessage(producerId, sequenceId, batchMessageContainer.numMessagesInBatch, batchMessageContainer.setBatchAndBuild(), compressedPayload); op.setBatchSizeByte(batchMessageContainer.currentBatchSizeBytes); batchMessageContainer.clear();
if (batchMessageContainer.hasSpaceInBatch(msg)) { batchMessageContainer.add(msg, callback); payload.release(); if (batchMessageContainer.numMessagesInBatch == maxNumMessagesInBatch
public ProducerImpl(PulsarClientImpl client, String topic, String producerName, ProducerConfiguration conf, CompletableFuture<Producer> producerCreatedFuture, int partitionIndex) { super(client, topic, conf, producerCreatedFuture); this.producerId = client.newProducerId(); this.producerName = producerName; this.partitionIndex = partitionIndex; this.pendingMessages = Queues.newArrayBlockingQueue(conf.getMaxPendingMessages()); this.pendingCallbacks = Queues.newArrayBlockingQueue(conf.getMaxPendingMessages()); this.semaphore = new Semaphore(conf.getMaxPendingMessages(), true); this.compressor = CompressionCodecProvider .getCompressionCodec(convertCompressionType(conf.getCompressionType())); if (conf.getSendTimeoutMs() > 0) { sendTimeout = client.timer().newTimeout(this, conf.getSendTimeoutMs(), TimeUnit.MILLISECONDS); } this.createProducerTimeout = System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs(); if (conf.getBatchingEnabled()) { this.maxNumMessagesInBatch = conf.getBatchingMaxMessages(); this.batchMessageContainer = new BatchMessageContainer(maxNumMessagesInBatch, convertCompressionType(conf.getCompressionType()), topic, producerName); } else { this.maxNumMessagesInBatch = 1; this.batchMessageContainer = null; } if (client.getConfiguration().getStatsIntervalSeconds() > 0) { stats = new ProducerStats(client, conf, this); } else { stats = ProducerStats.PRODUCER_STATS_DISABLED; } grabCnx(); }