continue; if (singleMessageMetadataBuilder.getCompactedOut()) { singleMessageMetadataBuilder.recycle(); messageId.getEntryId(), getPartitionIndex(), i, acker); final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, createEncryptionContext(msgMetadata), cnx, schema, redeliveryCount); if (possibleToDeadLetter != null) { singleMessageMetadataBuilder.recycle();
@Override public Optional<String> getKey() { if (singleMessageMetadata != null && singleMessageMetadata.hasPartitionKey()) { return Optional.of(singleMessageMetadata.getPartitionKey()); } else if (msgMetadata.get().hasPartitionKey()){ return Optional.of(msgMetadata.get().getPartitionKey()); } else { return Optional.empty(); } } }
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata.getDefaultInstance()) return this; if (!other.properties_.isEmpty()) { if (properties_.isEmpty()) { properties_ = other.properties_; bitField0_ = (bitField0_ & ~0x00000001); } else { ensurePropertiesIsMutable(); properties_.addAll(other.properties_); } } if (other.hasPartitionKey()) { setPartitionKey(other.getPartitionKey()); } if (other.hasPayloadSize()) { setPayloadSize(other.getPayloadSize()); } if (other.hasCompactedOut()) { setCompactedOut(other.getCompactedOut()); } if (other.hasEventTime()) { setEventTime(other.getEventTime()); } if (other.hasPartitionKeyB64Encoded()) { setPartitionKeyB64Encoded(other.getPartitionKeyB64Encoded()); } return this; }
continue; if (singleMessageMetadataBuilder.getCompactedOut()) { singleMessageMetadataBuilder.recycle(); messageId.getEntryId(), getPartitionIndex(), i, acker); final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, createEncryptionContext(msgMetadata), cnx, schema); if (possibleToDeadLetter != null) { singleMessageMetadataBuilder.recycle();
int messagesRetained = 0; SingleMessageMetadata.Builder emptyMetadataBuilder = SingleMessageMetadata.newBuilder().setCompactedOut(true); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); msg.getMessageIdData().getPartition(), i); if (!singleMessageMetadataBuilder.hasPartitionKey()) { messagesRetained++; Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, singleMessagePayload, batchBuffer); } else if (filter.test(singleMessageMetadataBuilder.getPartitionKey(), id) && singleMessagePayload.readableBytes() > 0) { messagesRetained++; Unpooled.EMPTY_BUFFER, batchBuffer); singleMessageMetadataBuilder.recycle(); singleMessagePayload.release(); emptyMetadataBuilder.recycle();
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata.getDefaultInstance()) return this; if (!other.properties_.isEmpty()) { if (properties_.isEmpty()) { properties_ = other.properties_; bitField0_ = (bitField0_ & ~0x00000001); } else { ensurePropertiesIsMutable(); properties_.addAll(other.properties_); } } if (other.hasPartitionKey()) { setPartitionKey(other.getPartitionKey()); } if (other.hasPayloadSize()) { setPayloadSize(other.getPayloadSize()); } if (other.hasCompactedOut()) { setCompactedOut(other.getCompactedOut()); } if (other.hasEventTime()) { setEventTime(other.getEventTime()); } if (other.hasPartitionKeyB64Encoded()) { setPartitionKeyB64Encoded(other.getPartitionKeyB64Encoded()); } return this; }
public static void receiveIndividualMessagesFromBatch(String topic, MessageMetadata msgMetadata, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, int partitionIndex, MessageProcessor processor) { int batchSize = msgMetadata.getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex, i, null); final MessageImpl<?> message = new MessageImpl<>( topic, batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, Optional.empty(), cnx, null); processor.process(batchMessageIdImpl, message, singleMessagePayload); singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
public static List<ImmutablePair<MessageId,String>> extractIdsAndKeys(RawMessage msg) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); int batchSize = metadata.getNumMessagesInBatch(); CompressionType compressionType = metadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); metadata.recycle(); List<ImmutablePair<MessageId,String>> idsAndKeys = new ArrayList<>(); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, 0, batchSize); MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); if (!singleMessageMetadataBuilder.getCompactedOut()) { idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey())); } singleMessageMetadataBuilder.recycle(); singleMessagePayload.release(); } uncompressedPayload.release(); return idsAndKeys; }
private List<Message<byte[]>> getIndividualMsgsFromBatch(String topic, String msgId, byte[] data, Map<String, String> properties) { List<Message<byte[]>> ret = new ArrayList<>(); int batchSize = Integer.parseInt(properties.get(BATCH_HEADER)); for (int i = 0; i < batchSize; i++) { String batchMsgId = msgId + ":" + i; PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf buf = Unpooled.wrappedBuffer(data); try { ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadataBuilder, i, batchSize); SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.build(); if (singleMessageMetadata.getPropertiesCount() > 0) { for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } } ret.add(new MessageImpl<>(topic, batchMsgId, properties, singleMessagePayload, Schema.BYTES)); } catch (Exception ex) { log.error("Exception occured while trying to get BatchMsgId: {}", batchMsgId, ex); } buf.release(); singleMessageMetadataBuilder.recycle(); } return ret; }
public static ByteBuf deSerializeSingleMessageInBatch(ByteBuf uncompressedPayload, PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, int index, int batchSize) throws IOException { int singleMetaSize = (int) uncompressedPayload.readUnsignedInt(); int writerIndex = uncompressedPayload.writerIndex(); int beginIndex = uncompressedPayload.readerIndex() + singleMetaSize; uncompressedPayload.writerIndex(beginIndex); ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(uncompressedPayload); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.mergeFrom(stream, null) .build(); int singleMessagePayloadSize = singleMessageMetadata.getPayloadSize(); uncompressedPayload.markReaderIndex(); ByteBuf singleMessagePayload = uncompressedPayload.slice(uncompressedPayload.readerIndex(), singleMessagePayloadSize); singleMessagePayload.retain(); uncompressedPayload.writerIndex(writerIndex); uncompressedPayload.resetReaderIndex(); // reader now points to beginning of payload read; so move it past message payload just read if (index < batchSize) { uncompressedPayload.readerIndex(uncompressedPayload.readerIndex() + singleMessagePayloadSize); } return singleMessagePayload; }
public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.MessageMetadata.Builder msgBuilder, ByteBuf payload, ByteBuf batchBuffer) { // build single message meta-data PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); if (msgBuilder.hasPartitionKey()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey()) .setPartitionKeyB64Encoded(msgBuilder.getPartitionKeyB64Encoded()); } if (!msgBuilder.getPropertiesList().isEmpty()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder .addAllProperties(msgBuilder.getPropertiesList()); } if (msgBuilder.hasEventTime()) { singleMessageMetadataBuilder.setEventTime(msgBuilder.getEventTime()); } try { return serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, payload, batchBuffer); } finally { singleMessageMetadataBuilder.recycle(); } }
public static ByteBuf serializeSingleMessageInBatchWithPayload( PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, ByteBuf payload, ByteBuf batchBuffer) { int payLoadSize = payload.readableBytes(); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.setPayloadSize(payLoadSize) .build(); // serialize meta-data size, meta-data and payload for single message in batch int singleMsgMetadataSize = singleMessageMetadata.getSerializedSize(); try { batchBuffer.writeInt(singleMsgMetadataSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(batchBuffer); singleMessageMetadata.writeTo(outStream); singleMessageMetadata.recycle(); outStream.recycle(); } catch (IOException e) { throw new RuntimeException(e); } return batchBuffer.writeBytes(payload); }
private static void receiveIndividualMessagesFromBatch(ReferenceCountedObject<MessageMetadata> msgMetadata, ByteBuf uncompressedPayload, long ledgerId, long entryId, MessageProcessor processor) { int batchSize = msgMetadata.get().getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } processor.process(RawMessageImpl.get(msgMetadata, singleMessageMetadataBuilder, singleMessagePayload, ledgerId, entryId, i)); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.MessageMetadata.Builder msgBuilder, ByteBuf payload, ByteBuf batchBuffer) { // build single message meta-data PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); if (msgBuilder.hasPartitionKey()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey()) .setPartitionKeyB64Encoded(msgBuilder.getPartitionKeyB64Encoded()); } if (!msgBuilder.getPropertiesList().isEmpty()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder .addAllProperties(msgBuilder.getPropertiesList()); } if (msgBuilder.hasEventTime()) { singleMessageMetadataBuilder.setEventTime(msgBuilder.getEventTime()); } try { return serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, payload, batchBuffer); } finally { singleMessageMetadataBuilder.recycle(); } }
public static ByteBuf serializeSingleMessageInBatchWithPayload( PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, ByteBuf payload, ByteBuf batchBuffer) { int payLoadSize = payload.readableBytes(); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.setPayloadSize(payLoadSize) .build(); // serialize meta-data size, meta-data and payload for single message in batch int singleMsgMetadataSize = singleMessageMetadata.getSerializedSize(); try { batchBuffer.writeInt(singleMsgMetadataSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(batchBuffer); singleMessageMetadata.writeTo(outStream); singleMessageMetadata.recycle(); outStream.recycle(); } catch (IOException e) { throw new RuntimeException(e); } return batchBuffer.writeBytes(payload); }
@Override public void release() { msgMetadata.release(); msgMetadata = null; if (singleMessageMetadata != null) { singleMessageMetadata.recycle(); singleMessageMetadata = null; } payload.release(); handle.recycle(this); }