public void makeMessageToCosumeAgain(List<MessageExt> msgs) { try { this.lockTreeMap.writeLock().lockInterruptibly(); try { for (MessageExt msg : msgs) { this.consumingMsgOrderlyTreeMap.remove(msg.getQueueOffset()); this.msgTreeMap.put(msg.getQueueOffset(), msg); } } finally { this.lockTreeMap.writeLock().unlock(); } } catch (InterruptedException e) { log.error("makeMessageToCosumeAgain exception", e); } }
int validMsgCnt = 0; for (MessageExt msg : msgs) { MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg); if (null == old) { validMsgCnt++; this.queueOffsetMax = msg.getQueueOffset(); msgSize.addAndGet(msg.getBody().length); String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET); if (property != null) { long accTotal = Long.parseLong(property) - messageExt.getQueueOffset(); if (accTotal > 0) { this.msgAccCnt = accTotal;
public long removeMessage(final List<MessageExt> msgs) { long result = -1; final long now = System.currentTimeMillis(); try { this.lockTreeMap.writeLock().lockInterruptibly(); this.lastConsumeTimestamp = now; try { if (!msgTreeMap.isEmpty()) { result = this.queueOffsetMax + 1; int removedCnt = 0; for (MessageExt msg : msgs) { MessageExt prev = msgTreeMap.remove(msg.getQueueOffset()); if (prev != null) { removedCnt--; msgSize.addAndGet(0 - msg.getBody().length); } } msgCount.addAndGet(removedCnt); if (!msgTreeMap.isEmpty()) { result = msgTreeMap.firstKey(); } } } finally { this.lockTreeMap.writeLock().unlock(); } } catch (Throwable t) { log.error("removeMessage exception", t); } return result; }
/** * Use this function while transaction msg is committed or rollback write a flag 'd' to operation queue for the * msg's offset * * @param messageExt Op message * @param messageQueue Op message queue * @return This method will always return true. */ private boolean addRemoveTagInTransactionOp(MessageExt messageExt, MessageQueue messageQueue) { Message message = new Message(TransactionalMessageUtil.buildOpTopic(), TransactionalMessageUtil.REMOVETAG, String.valueOf(messageExt.getQueueOffset()).getBytes(TransactionalMessageUtil.charset)); writeOp(message, messageQueue); return true; }
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext consumeConcurrentlyContext) { for (MessageExt msg : msgs) { msgIndex.getAndIncrement(); if (isDebug) { if (listenerName != null && !listenerName.isEmpty()) { logger.info(listenerName + ":" + msgIndex.get() + ":" + String.format("msgid:%s broker:%s queueId:%s offset:%s", msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset())); } else { logger.info(msg); } } msgBodys.addData(new String(msg.getBody())); originMsgs.addData(msg); if (originMsgIndex != null) { originMsgIndex.put(new String(msg.getBody()), msg); } } return consumeStatus; } }
private void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key) throws MQClientException, InterruptedException { admin.start(); QueryResult queryResult = admin.queryMessage(topic, key, 64, 0, Long.MAX_VALUE); System.out.printf("%-50s %4s %40s%n", "#Message ID", "#QID", "#Offset"); for (MessageExt msg : queryResult.getMessageList()) { System.out.printf("%-50s %4d %40d%n", msg.getMsgId(), msg.getQueueId(), msg.getQueueOffset()); } } }
log.info("send expire msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); try { this.lockTreeMap.writeLock().lockInterruptibly(); try { if (!msgTreeMap.isEmpty() && msg.getQueueOffset() == msgTreeMap.firstKey()) { try { removeMessage(Collections.singletonList(msg));
public MessageExtBrokerInner renewImmunityHalfMessageInner(MessageExt msgExt) { MessageExtBrokerInner msgInner = renewHalfMessageInner(msgExt); String queueOffsetFromPrepare = msgExt.getUserProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); if (null != queueOffsetFromPrepare) { MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, String.valueOf(queueOffsetFromPrepare)); } else { MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, String.valueOf(msgExt.getQueueOffset())); } msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); return msgInner; }
public boolean consumed(final MessageExt msg, final String group) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { ConsumeStats cstats = this.examineConsumeStats(group); ClusterInfo ci = this.examineBrokerClusterInfo(); Iterator<Entry<MessageQueue, OffsetWrapper>> it = cstats.getOffsetTable().entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, OffsetWrapper> next = it.next(); MessageQueue mq = next.getKey(); if (mq.getTopic().equals(msg.getTopic()) && mq.getQueueId() == msg.getQueueId()) { BrokerData brokerData = ci.getBrokerAddrTable().get(mq.getBrokerName()); if (brokerData != null) { String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); if (RemotingUtil.socketAddress2String(msg.getStoreHost()).equals(addr)) { if (next.getValue().getConsumerOffset() > msg.getQueueOffset()) { return true; } } } } } return false; }
long queueOffset = messageExt.getQueueOffset(); byteBuffer.putLong(queueOffset);
rocketmqPullConsumer.sendMessageBack(msg, 3); log.info("Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); ack(mq, pq, msg); } catch (Exception e) {
msg.getQueueOffset() );
msg.getQueueOffset() );
Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { if (queueOffset < miniOffset) { doneOpOffset.add(opMessageExt.getQueueOffset()); } else { removeMap.put(queueOffset, opMessageExt.getQueueOffset());
private RemotingCommand checkPrepareMessage(MessageExt msgExt, EndTransactionRequestHeader requestHeader) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); if (msgExt != null) { final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); if (!pgroupRead.equals(requestHeader.getProducerGroup())) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("The producer group wrong"); return response; } if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("The transaction state table offset wrong"); return response; } if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("The commit log offset wrong"); return response; } } else { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("Find prepared transaction message failed"); return response; } response.setCode(ResponseCode.SUCCESS); return response; }
public void sendCheckMessage(MessageExt msgExt) throws Exception { CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader(); checkTransactionStateRequestHeader.setCommitLogOffset(msgExt.getCommitLogOffset()); checkTransactionStateRequestHeader.setOffsetMsgId(msgExt.getMsgId()); checkTransactionStateRequestHeader.setMsgId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); checkTransactionStateRequestHeader.setTransactionId(checkTransactionStateRequestHeader.getMsgId()); checkTransactionStateRequestHeader.setTranStateTableOffset(msgExt.getQueueOffset()); msgExt.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); msgExt.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); msgExt.setStoreSize(0); String groupId = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); Channel channel = brokerController.getProducerManager().getAvaliableChannel(groupId); if (channel != null) { brokerController.getBroker2Client().checkProducerTransactionState(groupId, channel, checkTransactionStateRequestHeader, msgExt); } else { LOGGER.warn("Check transaction failed, channel is null. groupId={}", groupId); } }
private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { PutMessageResult putMessageResult = putBackToHalfQueueReturnResult(msgExt); if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { msgExt.setQueueOffset( putMessageResult.getAppendMessageResult().getLogicsOffset()); msgExt.setCommitLogOffset( putMessageResult.getAppendMessageResult().getWroteOffset()); msgExt.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); log.info( "Send check message, the offset={} restored in queueOffset={} " + "commitLogOffset={} " + "newMsgId={} realMsgId={} topic={}", offset, msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getMsgId(), msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX), msgExt.getTopic()); return true; } else { log.error( "PutBackToHalfQueueReturnResult write failed, topic: {}, queueId: {}, " + "msgId: {}", msgExt.getTopic(), msgExt.getQueueId(), msgExt.getMsgId()); return false; } }
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); } else { firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
public void putMessages(final MessageQueue mq, final List<MessageExt> msgs) { CachedQueue cachedQueue = this.mqCachedTable.get(mq); if (null == cachedQueue) { cachedQueue = new CachedQueue(); this.mqCachedTable.put(mq, cachedQueue); } for (MessageExt msg : msgs) { cachedQueue.getMsgCachedTable().put(msg.getQueueOffset(), msg); } }
public void putMessages(final MessageQueue mq, final List<MessageExt> msgs) { CachedQueue cachedQueue = this.mqCachedTable.get(mq); if (null == cachedQueue) { cachedQueue = new CachedQueue(); this.mqCachedTable.put(mq, cachedQueue); } for (MessageExt msg : msgs) { cachedQueue.getMsgCachedTable().put(msg.getQueueOffset(), msg); } }