PartitionedConsumerImpl(PulsarClientImpl client, String topic, String subscription, ConsumerConfiguration conf, int numPartitions, ExecutorService listenerExecutor, CompletableFuture<Consumer> subscribeFuture) { super(client, topic, subscription, conf, Math.max(numPartitions, conf.getReceiverQueueSize()), listenerExecutor, subscribeFuture); this.consumers = Lists.newArrayListWithCapacity(numPartitions); this.pausedConsumers = new ConcurrentLinkedQueue<>(); this.sharedQueueResumeThreshold = maxReceiverQueueSize / 2; this.numPartitions = numPartitions; stats = client.getConfiguration().getStatsIntervalSeconds() > 0 ? new ConsumerStats() : null; checkArgument(conf.getReceiverQueueSize() > 0, "Receiver queue size needs to be greater than 0 for Partitioned Topics"); start(); }
@Override public void operationComplete(Future<Void> future) throws Exception { if (future.isSuccess()) { if (ackType == AckType.Individual) { unAckedMessageTracker.remove(msgId); // increment counter by 1 for non-batch msg if (!(messageId instanceof BatchMessageIdImpl)) { stats.incrementNumAcksSent(1); } } else if (ackType == AckType.Cumulative) { stats.incrementNumAcksSent(unAckedMessageTracker.removeMessagesTill(msgId)); } if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Successfully acknowledged message - {}, acktype {}", subscription, topic, consumerName, messageId, ackType); } ackFuture.complete(null); } else { stats.incrementNumAcksFailed(); ackFuture.completeExceptionally(new PulsarClientException(future.cause())); } } });
@Override protected Message internalReceive(int timeout, TimeUnit unit) throws PulsarClientException { Message message; try { message = incomingMessages.poll(timeout, unit); if (message != null) { messageProcessed(message); } return message; } catch (InterruptedException e) { Thread.currentThread().interrupt(); stats.incrementNumReceiveFailed(); throw new PulsarClientException(e); } }
@Override public synchronized ConsumerStats getStats() { if (stats == null) { return null; } stats.reset(); for (int i = 0; i < numPartitions; i++) { stats.updateCumulativeStats(consumers.get(i).getStats()); } return stats; }
stats.updateNumMsgsReceived(message); return message; } catch (InterruptedException e) { Thread.currentThread().interrupt(); stats.incrementNumReceiveFailed(); throw new PulsarClientException(e); } finally {
@Override protected CompletableFuture<Void> doAcknowledge(MessageId messageId, AckType ackType) { checkArgument(messageId instanceof MessageIdImpl); if (getState() != State.Ready && getState() != State.Connecting) { stats.incrementNumAcksFailed(); return FutureUtil.failedFuture(new PulsarClientException("Consumer not ready. State: " + getState())); } if (messageId instanceof BatchMessageIdImpl) { if (markAckForBatchMessage((BatchMessageIdImpl) messageId, ackType)) { // all messages in batch have been acked so broker can be acked via sendAcknowledge() if (log.isDebugEnabled()) { log.debug("[{}] [{}] acknowledging message - {}, acktype {}", subscription, consumerName, messageId, ackType); } } else { // other messages in batch are still pending ack. return CompletableFuture.completedFuture(null); } } // if we got a cumulative ack on non batch message, check if any earlier batch messages need to be removed // from batch message tracker if (ackType == AckType.Cumulative && !(messageId instanceof BatchMessageIdImpl)) { updateBatchAckTracker((MessageIdImpl) messageId, ackType); } return sendAcknowledge(messageId, ackType); }
stats.incrementNumAcksSent(batchSize);
Timeout timeout = stats.getStatTimeout(); if (timeout != null) { timeout.cancel();
public ConsumerStats(PulsarClientImpl pulsarClient, ConsumerConfiguration conf, ConsumerImpl consumer) { this.pulsarClient = pulsarClient; this.consumer = consumer; this.statsIntervalSeconds = pulsarClient.getConfiguration().getStatsIntervalSeconds(); numMsgsReceived = new LongAdder(); numBytesReceived = new LongAdder(); numReceiveFailed = new LongAdder(); numAcksSent = new LongAdder(); numAcksFailed = new LongAdder(); totalMsgsReceived = new LongAdder(); totalBytesReceived = new LongAdder(); totalReceiveFailed = new LongAdder(); totalAcksSent = new LongAdder(); totalAcksFailed = new LongAdder(); throughputFormat = new DecimalFormat("0.00"); init(conf); }
/** * Record the event that one message has been processed by the application. * * Periodically, it sends a Flow command to notify the broker that it can push more messages */ protected synchronized void messageProcessed(Message msg) { ClientCnx currentCnx = cnx(); ClientCnx msgCnx = ((MessageImpl) msg).getCnx(); lastDequeuedMessage = (MessageIdImpl) msg.getMessageId(); if (msgCnx != currentCnx) { // The processed message did belong to the old queue that was cleared after reconnection. return; } increaseAvailablePermits(currentCnx); stats.updateNumMsgsReceived(msg); if (conf.getAckTimeoutMillis() != 0) { // reset timer for messages that are received by the client MessageIdImpl id = (MessageIdImpl) msg.getMessageId(); if (id instanceof BatchMessageIdImpl) { id = new MessageIdImpl(id.getLedgerId(), id.getEntryId(), getPartitionIndex()); } unAckedMessageTracker.add(id); } }
stats.incrementNumAcksFailed(); ackFuture .completeExceptionally(new PulsarClientException("Not connected to broker. State: " + getState()));
@Override protected Message internalReceive() throws PulsarClientException { if (conf.getReceiverQueueSize() == 0) { checkArgument(zeroQueueLock != null, "Receiver queue size can't be modified"); zeroQueueLock.writeLock().lock(); try { return fetchSingleMessageFromBroker(); } finally { zeroQueueLock.writeLock().unlock(); } } Message message; try { message = incomingMessages.take(); messageProcessed(message); return message; } catch (InterruptedException e) { Thread.currentThread().interrupt(); stats.incrementNumReceiveFailed(); throw new PulsarClientException(e); } }
this.batchMessageAckTracker = new ConcurrentSkipListMap<>(); if (client.getConfiguration().getStatsIntervalSeconds() > 0) { stats = new ConsumerStats(client, conf, this); } else { stats = ConsumerStats.CONSUMER_STATS_DISABLED;
private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentCnx, ValidationError validationError) { log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId()); ByteBuf cmd = Commands.newAck(consumerId, messageId.getLedgerId(), messageId.getEntryId(), AckType.Individual, validationError); currentCnx.ctx().writeAndFlush(cmd, currentCnx.ctx().voidPromise()); increaseAvailablePermits(currentCnx); stats.incrementNumReceiveFailed(); }