public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); AppendMessageResult result; final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag()); if (messageExtBatch.getDelayTimeLevel() > 0) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch)); messageExtBatch.setStoreTimestamp(beginLockTimestamp); log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); if (null == mappedFile) { log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); log.warn("[NOTIFYME]putMessages in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, messageExtBatch.getBody().length, result); storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(result.getMsgNum()); storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(result.getWroteBytes());
return response; MessageExtBatch messageExtBatch = new MessageExtBatch(); messageExtBatch.setTopic(requestHeader.getTopic()); messageExtBatch.setQueueId(queueIdInt); sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG; messageExtBatch.setSysFlag(sysFlag); messageExtBatch.setFlag(requestHeader.getFlag()); MessageAccessor.setProperties(messageExtBatch, MessageDecoder.string2messageProperties(requestHeader.getProperties())); messageExtBatch.setBody(request.getBody()); messageExtBatch.setBornTimestamp(requestHeader.getBornTimestamp()); messageExtBatch.setBornHost(ctx.channel().remoteAddress()); messageExtBatch.setStoreHost(this.getStoreHost()); messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
public ByteBuffer encode(final MessageExtBatch messageExtBatch) { ByteBuffer messagesByteBuff = messageExtBatch.wrap(); while (messagesByteBuff.hasRemaining()) { messagesByteBuff.position(propertiesPos + propertiesLen); final byte[] topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); this.msgBatchMemory.putInt(messageExtBatch.getQueueId()); this.msgBatchMemory.putInt(messageExtBatch.getSysFlag()); this.msgBatchMemory.putLong(messageExtBatch.getBornTimestamp()); this.msgBatchMemory.put(messageExtBatch.getBornHostBytes(hostHolder)); this.msgBatchMemory.putLong(messageExtBatch.getStoreTimestamp()); this.msgBatchMemory.put(messageExtBatch.getStoreHostBytes(hostHolder)); this.msgBatchMemory.putInt(messageExtBatch.getReconsumeTimes());
keyBuilder.append(messageExtBatch.getTopic()); keyBuilder.append('-'); keyBuilder.append(messageExtBatch.getQueueId()); String key = keyBuilder.toString(); Long queueOffset = CommitLog.this.topicQueueTable.get(key); msgIdBuilder.setLength(0); final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff(); this.resetByteBuffer(hostHolder, 8); ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes(hostHolder); messagesByteBuff.mark(); while (messagesByteBuff.hasRemaining()) { return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); messagesByteBuff.limit(totalMsgLen); byteBuffer.put(messagesByteBuff); messageExtBatch.setEncodedBuff(null); AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, totalMsgLen, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); result.setMsgNum(msgNum); CommitLog.this.topicQueueTable.put(key, queueOffset);
ByteBuffer messagesByteBuff = messageExtBatch.wrap(); while (messagesByteBuff.hasRemaining()) { int queueId; int topicIdx = 0; if (messageExtBatch.isMultiTopic()) { if (topicIdx >= messageExtBatch.getTopics().length) { CommitLog.log.warn("topic index exceeded, total topic size: " + messageExtBatch.getTopics().length + ", topic index size: " + topicIdx); throw new RuntimeException("topic index size exceeded"); //TO DO handle this exception topicData = messageExtBatch.getTopics()[topicIdx].getBytes(MessageDecoder.CHARSET_UTF8); topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); queueId = messageExtBatch.getQueueId(); this.msgBatchMemory.putInt(messageExtBatch.getSysFlag()); this.msgBatchMemory.putLong(messageExtBatch.getBornTimestamp()); this.msgBatchMemory.put(messageExtBatch.getBornHostBytes(hostHolder)); this.msgBatchMemory.putLong(messageExtBatch.getStoreTimestamp()); this.msgBatchMemory.put(messageExtBatch.getStoreHostBytes(hostHolder)); this.msgBatchMemory.putInt(messageExtBatch.getReconsumeTimes()); messageExtBatch.setStoreSize(maxMessageSize);
if ((messageExtBatch.getStoreSize() + END_FILE_MIN_BLANK_LENGTH) > maxBlank) { this.resetByteBuffer(this.msgStoreItemMemory, 8); return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), 0L, CommitLog.this.defaultMessageStore.now() - beginTimeMills); ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff(); messagesByteBuff.mark(); while (messagesByteBuff.hasRemaining()) { messagesByteBuff.position(msgPos + 20); final int topicIdx = (int) messagesByteBuff.getLong(); String topic = messageExtBatch.getTopics()[topicIdx]; messagesByteBuff.limit(totalMsgLen); byteBuffer.put(messagesByteBuff); messageExtBatch.setEncodedBuff(null); AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, totalMsgLen, messageExtBatch.getStoreTimestamp(), 0 , CommitLog.this.defaultMessageStore.now() - beginTimeMills); result.setMsgNum(msgNum);
if (messageExtBatch.isMultiTopic()) { for (String topic : messageExtBatch.getTopics()) { if (topic.length() > Byte.MAX_VALUE) { log.warn("PutMessages topic length too long " + topic.length()); if (messageExtBatch.getTopic().length() > Byte.MAX_VALUE) { log.warn("PutMessages topic length too long " + messageExtBatch.getTopic().length()); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); if (messageExtBatch.getBody().length > messageStoreConfig.getMaxMessageSize()) { log.warn("PutMessages body length too long " + messageExtBatch.getBody().length); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); log.warn("not in lock eclipse time(ms)={}, bodyLength={}", eclipseTime, messageExtBatch.getBody().length);
if (messageExtBatch.getTopic().length() > Byte.MAX_VALUE) { log.warn("PutMessages topic length too long " + messageExtBatch.getTopic().length()); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); if (messageExtBatch.getBody().length > messageStoreConfig.getMaxMessageSize()) { log.warn("PutMessages body length too long " + messageExtBatch.getBody().length); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); log.warn("not in lock eclipse time(ms)={}, bodyLength={}", eclipseTime, messageExtBatch.getBody().length);
public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, final int maxBlank, final MessageExtBatch messageExtBatch) { if (messageExtBatch.isMultiTopic()) { return doAppendMultiTopic(fileFromOffset, byteBuffer, maxBlank, messageExtBatch); keyBuilder.append(messageExtBatch.getTopic()); keyBuilder.append('-'); keyBuilder.append(messageExtBatch.getQueueId()); String key = keyBuilder.toString(); Long queueOffset = CommitLog.this.topicQueueTable.get(key); msgIdBuilder.setLength(0); final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff(); this.resetByteBuffer(hostHolder, 8); ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes(hostHolder); messagesByteBuff.mark(); while (messagesByteBuff.hasRemaining()) { return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); messagesByteBuff.limit(totalMsgLen); byteBuffer.put(messagesByteBuff); messageExtBatch.setEncodedBuff(null); AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, totalMsgLen, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); result.setMsgNum(msgNum); CommitLog.this.topicQueueTable.put(key, queueOffset);
if (messageExtBatch.getTopic().length() > Byte.MAX_VALUE) { log.warn("PutMessages topic length too long " + messageExtBatch.getTopic().length()); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); if (messageExtBatch.getBody().length > messageStoreConfig.getMaxMessageSize()) { log.warn("PutMessages body length too long " + messageExtBatch.getBody().length); return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); log.warn("not in lock eclipse time(ms)={}, bodyLength={}", eclipseTime, messageExtBatch.getBody().length);
return response; MessageExtBatch messageExtBatch = new MessageExtBatch(); messageExtBatch.setTopic(requestHeader.getTopic()); messageExtBatch.setQueueId(queueIdInt); if (topics != null) { messageExtBatch.setMultiTopic(true); messageExtBatch.setTopics(topics); sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG; messageExtBatch.setSysFlag(sysFlag); messageExtBatch.setFlag(requestHeader.getFlag()); MessageAccessor.setProperties(messageExtBatch, MessageDecoder.string2messageProperties(requestHeader.getProperties())); messageExtBatch.setBody(request.getBody()); messageExtBatch.setBornTimestamp(requestHeader.getBornTimestamp()); messageExtBatch.setBornHost(ctx.channel().remoteAddress()); messageExtBatch.setStoreHost(this.getStoreHost()); messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); AppendMessageResult result; final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag()); if (messageExtBatch.getDelayTimeLevel() > 0) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); messageExtBatch.setEncodedBuff(encodedBuffer); } catch (RuntimeException e) { log.error("encode error:", e); messageExtBatch.setStoreTimestamp(beginLockTimestamp); log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); if (null == mappedFile) { log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); log.warn("[NOTIFYME]putMessages in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, messageExtBatch.getBody().length, result); storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(result.getMsgNum()); storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(result.getWroteBytes());
public ByteBuffer encode(final MessageExtBatch messageExtBatch) { ByteBuffer messagesByteBuff = messageExtBatch.wrap(); while (messagesByteBuff.hasRemaining()) { messagesByteBuff.position(propertiesPos + propertiesLen); final byte[] topicData = messageExtBatch.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); this.msgBatchMemory.putInt(messageExtBatch.getQueueId()); this.msgBatchMemory.putInt(messageExtBatch.getSysFlag()); this.msgBatchMemory.putLong(messageExtBatch.getBornTimestamp()); this.msgBatchMemory.put(messageExtBatch.getBornHostBytes(hostHolder)); this.msgBatchMemory.putLong(messageExtBatch.getStoreTimestamp()); this.msgBatchMemory.put(messageExtBatch.getStoreHostBytes(hostHolder)); this.msgBatchMemory.putInt(messageExtBatch.getReconsumeTimes());
keyBuilder.append(messageExtBatch.getTopic()); keyBuilder.append('-'); keyBuilder.append(messageExtBatch.getQueueId()); String key = keyBuilder.toString(); Long queueOffset = CommitLog.this.topicQueueTable.get(key); msgIdBuilder.setLength(0); final long beginTimeMills = CommitLog.this.defaultMessageStore.now(); ByteBuffer messagesByteBuff = messageExtBatch.getEncodedBuff(); this.resetByteBuffer(hostHolder, 8); ByteBuffer storeHostBytes = messageExtBatch.getStoreHostBytes(hostHolder); messagesByteBuff.mark(); while (messagesByteBuff.hasRemaining()) { return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); messagesByteBuff.limit(totalMsgLen); byteBuffer.put(messagesByteBuff); messageExtBatch.setEncodedBuff(null); AppendMessageResult result = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, totalMsgLen, msgIdBuilder.toString(), messageExtBatch.getStoreTimestamp(), beginQueueOffset, CommitLog.this.defaultMessageStore.now() - beginTimeMills); result.setMsgNum(msgNum); CommitLog.this.topicQueueTable.put(key, queueOffset);
return response; MessageExtBatch messageExtBatch = new MessageExtBatch(); messageExtBatch.setTopic(requestHeader.getTopic()); messageExtBatch.setQueueId(queueIdInt); sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG; messageExtBatch.setSysFlag(sysFlag); messageExtBatch.setFlag(requestHeader.getFlag()); MessageAccessor.setProperties(messageExtBatch, MessageDecoder.string2messageProperties(requestHeader.getProperties())); messageExtBatch.setBody(request.getBody()); messageExtBatch.setBornTimestamp(requestHeader.getBornTimestamp()); messageExtBatch.setBornHost(ctx.channel().remoteAddress()); messageExtBatch.setStoreHost(this.getStoreHost()); messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); AppendMessageResult result; final int tranType = MessageSysFlag.getTransactionValue(messageExtBatch.getSysFlag()); if (messageExtBatch.getDelayTimeLevel() > 0) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch)); messageExtBatch.setStoreTimestamp(beginLockTimestamp); log.error("Create mapped file1 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); if (null == mappedFile) { log.error("Create mapped file2 error, topic: {} clientAddr: {}", messageExtBatch.getTopic(), messageExtBatch.getBornHostString()); beginTimeInLock = 0; return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); log.warn("[NOTIFYME]putMessages in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, messageExtBatch.getBody().length, result); storeStatsService.getSinglePutMessageTopicTimesTotal(messageExtBatch.getTopic()).addAndGet(result.getMsgNum()); storeStatsService.getSinglePutMessageTopicSizeTotal(messageExtBatch.getTopic()).addAndGet(result.getWroteBytes());