public static byte[] encode(MessageExt messageExt, boolean needCompress) throws Exception { byte[] body = messageExt.getBody(); byte[] topics = messageExt.getTopic().getBytes(CHARSET_UTF8); byte topicLen = (byte) topics.length; String properties = messageProperties2String(messageExt.getProperties()); byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); short propertiesLength = (short) propertiesBytes.length; int sysFlag = messageExt.getSysFlag(); byte[] newBody = messageExt.getBody(); if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { newBody = UtilAll.compress(body, 5); int storeSize = messageExt.getStoreSize(); ByteBuffer byteBuffer; if (storeSize > 0) { int bodyCRC = messageExt.getBodyCRC(); byteBuffer.putInt(bodyCRC); int queueId = messageExt.getQueueId(); byteBuffer.putInt(queueId); int flag = messageExt.getFlag(); byteBuffer.putInt(flag); long queueOffset = messageExt.getQueueOffset(); byteBuffer.putLong(queueOffset); long physicOffset = messageExt.getCommitLogOffset(); byteBuffer.putLong(physicOffset);
String msgId = msg.getMsgId(); if (msg instanceof MessageClientExt) { msgId = ((MessageClientExt) msg).getOffsetMsgId(); msg.getTopic() ); "[" + msg.getTags() + "]" ); "[" + msg.getKeys() + "]" ); msg.getQueueId() ); msg.getQueueOffset() ); msg.getCommitLogOffset() ); msg.getReconsumeTimes() ); UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) ); UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp())
public static void printMessage(final List<MessageExt> msgs, final String charsetName, boolean printBody) { for (MessageExt msg : msgs) { try { System.out.printf("MSGID: %s %s BODY: %s%n", msg.getMsgId(), msg.toString(), printBody ? new String(msg.getBody(), charsetName) : "NOT PRINT BODY"); } catch (UnsupportedEncodingException e) { } } }
public void resetRetryTopic(final List<MessageExt> msgs) { final String groupTopic = MixAll.getRetryTopic(consumerGroup); for (MessageExt msg : msgs) { String retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); if (retryTopic != null && groupTopic.equals(msg.getTopic())) { msg.setTopic(retryTopic); } } }
public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { try { String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); } catch (Exception e) { log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); String originMsgId = MessageAccessor.getOriginMessageId(msg); MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); newMsg.setFlag(msg.getFlag()); MessageAccessor.setProperties(newMsg, msg.getProperties()); MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); this.mQClientFactory.getDefaultMQProducer().send(newMsg); } }
final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); if (null == retryTopic) { MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic()); msgExt.setWaitStoreMsgOK(false); if (msgExt.getReconsumeTimes() >= maxReconsumeTimes || delayLevel < 0) { newTopic = MixAll.getDLQTopic(requestHeader.getGroup()); delayLevel = 3 + msgExt.getReconsumeTimes(); msgExt.setDelayTimeLevel(delayLevel); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags())); msgInner.setSysFlag(msgExt.getSysFlag()); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1); MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId); switch (putMessageResult.getPutMessageStatus()) {
private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_TOPIC)); msgInner.setQueueId(Integer.parseInt(msgExt.getUserProperty(MessageConst.PROPERTY_REAL_QUEUE_ID))); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(msgExt.getStoreHost()); msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); msgInner.setWaitStoreMsgOK(false); msgInner.setTransactionId(msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX)); msgInner.setSysFlag(msgExt.getSysFlag()); TopicFilterType topicFilterType = (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG : TopicFilterType.SINGLE_TAG; long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_TOPIC); MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_REAL_QUEUE_ID); return msgInner; }
public static BytesMessage msgConvert(org.apache.rocketmq.common.message.MessageExt rmqMsg) { BytesMessage omsMsg = new BytesMessageImpl(); omsMsg.setBody(rmqMsg.getBody()); KeyValue headers = omsMsg.sysHeaders(); KeyValue properties = omsMsg.userHeaders(); final Set<Map.Entry<String, String>> entries = rmqMsg.getProperties().entrySet(); for (final Map.Entry<String, String> entry : entries) { if (isOMSHeader(entry.getKey())) { headers.put(entry.getKey(), entry.getValue()); } else { properties.put(entry.getKey(), entry.getValue()); } } omsMsg.putSysHeaders(BuiltinKeys.MESSAGE_ID, rmqMsg.getMsgId()); omsMsg.putSysHeaders(BuiltinKeys.DESTINATION, rmqMsg.getTopic()); omsMsg.putSysHeaders(BuiltinKeys.SEARCH_KEYS, rmqMsg.getKeys()); omsMsg.putSysHeaders(BuiltinKeys.BORN_HOST, String.valueOf(rmqMsg.getBornHost())); omsMsg.putSysHeaders(BuiltinKeys.BORN_TIMESTAMP, rmqMsg.getBornTimestamp()); omsMsg.putSysHeaders(BuiltinKeys.STORE_HOST, String.valueOf(rmqMsg.getStoreHost())); omsMsg.putSysHeaders(BuiltinKeys.STORE_TIMESTAMP, rmqMsg.getStoreTimestamp()); return omsMsg; }
public MessageExtBrokerInner renewHalfMessageInner(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(msgExt.getTopic()); msgInner.setBody(msgExt.getBody()); msgInner.setQueueId(msgExt.getQueueId()); msgInner.setMsgId(msgExt.getMsgId()); msgInner.setSysFlag(msgExt.getSysFlag()); msgInner.setTags(msgExt.getTags()); msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(msgInner.getTags())); MessageAccessor.setProperties(msgInner, msgExt.getProperties()); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(msgExt.getStoreHost()); msgInner.setWaitStoreMsgOK(false); return msgInner; }
private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException { int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag()); if (msg.getBody() != null) { if (msg.getBody().length >= this.filtersrvController.getFiltersrvConfig().getCompressMsgBodyOverHowmuch()) { byte[] data = UtilAll.compress(msg.getBody(), this.filtersrvController.getFiltersrvConfig().getZipCompressLevel()); if (data != null) { msg.setBody(data); sysFlag |= MessageSysFlag.COMPRESSED_FLAG; final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0; byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET); final int topicLength = topicData.length; String properties = MessageDecoder.messageProperties2String(msg.getProperties()); byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET); final int propertiesLength = propertiesData.length; msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody())); msgStoreItemMemory.putInt(msgInner.getQueueId()); msgStoreItemMemory.putInt(msgInner.getFlag()); msgStoreItemMemory.putLong(msgInner.getQueueOffset()); msgStoreItemMemory.putLong(msgInner.getCommitLogOffset()); msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); msgStoreItemMemory.put(msgInner.getBornHostBytes());
public boolean consumed(final MessageExt msg, final String group) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { ConsumeStats cstats = this.examineConsumeStats(group); ClusterInfo ci = this.examineBrokerClusterInfo(); Iterator<Entry<MessageQueue, OffsetWrapper>> it = cstats.getOffsetTable().entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, OffsetWrapper> next = it.next(); MessageQueue mq = next.getKey(); if (mq.getTopic().equals(msg.getTopic()) && mq.getQueueId() == msg.getQueueId()) { BrokerData brokerData = ci.getBrokerAddrTable().get(mq.getBrokerName()); if (brokerData != null) { String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); if (RemotingUtil.socketAddress2String(msg.getStoreHost()).equals(addr)) { if (next.getValue().getConsumerOffset() > msg.getQueueOffset()) { return true; } } } } } return false; }
public boolean sendMessageBack(final MessageExt msg) { try { // max reconsume times exceeded then send to dead letter queue. Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); String originMsgId = MessageAccessor.getOriginMessageId(msg); MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); newMsg.setFlag(msg.getFlag()); MessageAccessor.setProperties(newMsg, msg.getProperties()); MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes())); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); this.defaultMQPushConsumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(newMsg); return true; } catch (Exception e) { log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e); } return false; }
private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setBody(msgExt.getBody()); msgInner.setFlag(msgExt.getFlag()); MessageAccessor.setProperties(msgInner, msgExt.getProperties()); TopicFilterType topicFilterType = (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG : TopicFilterType.SINGLE_TAG; long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); msgInner.setTagsCode(tagsCodeValue); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); msgInner.setSysFlag(msgExt.getSysFlag()); msgInner.setBornTimestamp(msgExt.getBornTimestamp()); msgInner.setBornHost(msgExt.getBornHost()); msgInner.setStoreHost(msgExt.getStoreHost()); msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); msgInner.setWaitStoreMsgOK(false); MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); msgInner.setTopic(msgExt.getTopic()); msgInner.setQueueId(msgExt.getQueueId()); return msgInner; } }
public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext consumeConcurrentlyContext) { for (MessageExt msg : msgs) { msgIndex.getAndIncrement(); if (isDebug) { if (listenerName != null && !listenerName.isEmpty()) { logger.info(listenerName + ":" + msgIndex.get() + ":" + String.format("msgid:%s broker:%s queueId:%s offset:%s", msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset())); } else { logger.info(msg); } } msgBodys.addData(new String(msg.getBody())); originMsgs.addData(msg); if (originMsgIndex != null) { originMsgIndex.put(new String(msg.getBody()), msg); } } return consumeStatus; } }
continue; String regionId = msg.getProperty(MessageConst.PROPERTY_MSG_REGION); String traceOn = msg.getProperty(MessageConst.PROPERTY_TRACE_SWITCH); traceBean.setTopic(msg.getTopic());// traceBean.setMsgId(msg.getMsgId());// traceBean.setTags(msg.getTags());// traceBean.setKeys(msg.getKeys());// traceBean.setStoreTime(msg.getStoreTimestamp());// traceBean.setBodyLength(msg.getStoreSize());// traceBean.setRetryTimes(msg.getReconsumeTimes());//
private boolean putBackHalfMsgQueue(MessageExt msgExt, long offset) { PutMessageResult putMessageResult = putBackToHalfQueueReturnResult(msgExt); if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { msgExt.setQueueOffset( putMessageResult.getAppendMessageResult().getLogicsOffset()); msgExt.setCommitLogOffset( putMessageResult.getAppendMessageResult().getWroteOffset()); msgExt.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); log.info( "Send check message, the offset={} restored in queueOffset={} " + "commitLogOffset={} " + "newMsgId={} realMsgId={} topic={}", offset, msgExt.getQueueOffset(), msgExt.getCommitLogOffset(), msgExt.getMsgId(), msgExt.getUserProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX), msgExt.getTopic()); return true; } else { log.error( "PutBackToHalfQueueReturnResult write failed, topic: {}, queueId: {}, " + "msgId: {}", msgExt.getTopic(), msgExt.getQueueId(), msgExt.getMsgId()); return false; } }
public static JmsBaseMessage convert2JMSMessage(MessageExt msg) throws Exception { JmsBaseMessage message; if (MsgConvertUtil.MSGMODEL_BYTES.equals( msg.getUserProperty(MsgConvertUtil.JMS_MSGMODEL))) { message = new JmsBytesMessage(msg.getBody()); msg.getUserProperty(MsgConvertUtil.JMS_MSGMODEL))) { message = new JmsObjectMessage(MsgConvertUtil.objectDeserialize(msg.getBody())); msg.getUserProperty(MsgConvertUtil.JMS_MSGMODEL))) { message = new JmsTextMessage(MsgConvertUtil.bytes2String(msg.getBody(), Charsets.UTF_8.toString())); message = new JmsBytesMessage(msg.getBody()); message.setHeader(JmsBaseConstant.JMS_MESSAGE_ID, "ID:" + msg.getMsgId()); if (msg.getReconsumeTimes() > 0) { message.setHeader(JmsBaseConstant.JMS_REDELIVERED, Boolean.TRUE); Map<String, String> propertiesMap = msg.getProperties(); if (propertiesMap != null) { for (String properName : propertiesMap.keySet()) {
@Override public LocalTransactionState checkLocalTransactionState(MessageExt msg) { System.out.println("TransactionCheckListener: " + new String(msg.getBody())); return LocalTransactionState.COMMIT_MESSAGE; }
Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { if (queueOffset < miniOffset) { doneOpOffset.add(opMessageExt.getQueueOffset()); } else { removeMap.put(queueOffset, opMessageExt.getQueueOffset());