.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize);
MessageImpl(String topic, BatchMessageIdImpl batchMessageIdImpl, MessageMetadata msgMetadata, PulsarApi.SingleMessageMetadata singleMessageMetadata, ByteBuf payload, Optional<EncryptionContext> encryptionCtx, ClientCnx cnx, Schema<T> schema, int redeliveryCount) { this.msgMetadataBuilder = MessageMetadata.newBuilder(msgMetadata); this.messageId = batchMessageIdImpl; this.topic = topic; this.cnx = cnx; this.redeliveryCount = redeliveryCount; this.payload = Unpooled.copiedBuffer(payload); this.encryptionCtx = encryptionCtx; if (singleMessageMetadata.getPropertiesCount() > 0) { Map<String, String> properties = Maps.newTreeMap(); for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } this.properties = Collections.unmodifiableMap(properties); } else { properties = Collections.emptyMap(); } if (singleMessageMetadata.hasPartitionKey()) { msgMetadataBuilder.setPartitionKeyB64Encoded(singleMessageMetadata.getPartitionKeyB64Encoded()); msgMetadataBuilder.setPartitionKey(singleMessageMetadata.getPartitionKey()); } if (singleMessageMetadata.hasEventTime()) { msgMetadataBuilder.setEventTime(singleMessageMetadata.getEventTime()); } this.schema = schema; }
.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize);
int messagesRetained = 0; SingleMessageMetadata.Builder emptyMetadataBuilder = SingleMessageMetadata.newBuilder().setCompactedOut(true); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder,
public static void receiveIndividualMessagesFromBatch(String topic, MessageMetadata msgMetadata, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, int partitionIndex, MessageProcessor processor) { int batchSize = msgMetadata.getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex, i, null); final MessageImpl<?> message = new MessageImpl<>( topic, batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, Optional.empty(), cnx, null); processor.process(batchMessageIdImpl, message, singleMessagePayload); singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
private List<Message<byte[]>> getIndividualMsgsFromBatch(String topic, String msgId, byte[] data, Map<String, String> properties) { List<Message<byte[]>> ret = new ArrayList<>(); int batchSize = Integer.parseInt(properties.get(BATCH_HEADER)); for (int i = 0; i < batchSize; i++) { String batchMsgId = msgId + ":" + i; PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf buf = Unpooled.wrappedBuffer(data); try { ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadataBuilder, i, batchSize); SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.build(); if (singleMessageMetadata.getPropertiesCount() > 0) { for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } } ret.add(new MessageImpl<>(topic, batchMsgId, properties, singleMessagePayload, Schema.BYTES)); } catch (Exception ex) { log.error("Exception occured while trying to get BatchMsgId: {}", batchMsgId, ex); } buf.release(); singleMessageMetadataBuilder.recycle(); } return ret; }
public static List<ImmutablePair<MessageId,String>> extractIdsAndKeys(RawMessage msg) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); int batchSize = metadata.getNumMessagesInBatch(); CompressionType compressionType = metadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); metadata.recycle(); List<ImmutablePair<MessageId,String>> idsAndKeys = new ArrayList<>(); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, 0, batchSize); MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); if (!singleMessageMetadataBuilder.getCompactedOut()) { idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey())); } singleMessageMetadataBuilder.recycle(); singleMessagePayload.release(); } uncompressedPayload.release(); return idsAndKeys; }
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata.getDefaultInstance()) return this; if (!other.properties_.isEmpty()) { if (properties_.isEmpty()) { properties_ = other.properties_; bitField0_ = (bitField0_ & ~0x00000001); } else { ensurePropertiesIsMutable(); properties_.addAll(other.properties_); } } if (other.hasPartitionKey()) { setPartitionKey(other.getPartitionKey()); } if (other.hasPayloadSize()) { setPayloadSize(other.getPayloadSize()); } if (other.hasCompactedOut()) { setCompactedOut(other.getCompactedOut()); } if (other.hasEventTime()) { setEventTime(other.getEventTime()); } if (other.hasPartitionKeyB64Encoded()) { setPartitionKeyB64Encoded(other.getPartitionKeyB64Encoded()); } return this; }
MessageImpl(String topic, BatchMessageIdImpl batchMessageIdImpl, MessageMetadata msgMetadata, PulsarApi.SingleMessageMetadata singleMessageMetadata, ByteBuf payload, Optional<EncryptionContext> encryptionCtx, ClientCnx cnx, Schema<T> schema) { this.msgMetadataBuilder = MessageMetadata.newBuilder(msgMetadata); this.messageId = batchMessageIdImpl; this.topic = topic; this.cnx = cnx; this.payload = Unpooled.copiedBuffer(payload); this.encryptionCtx = encryptionCtx; if (singleMessageMetadata.getPropertiesCount() > 0) { Map<String, String> properties = Maps.newTreeMap(); for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } this.properties = Collections.unmodifiableMap(properties); } else { properties = Collections.emptyMap(); } if (singleMessageMetadata.hasPartitionKey()) { msgMetadataBuilder.setPartitionKeyB64Encoded(singleMessageMetadata.getPartitionKeyB64Encoded()); msgMetadataBuilder.setPartitionKey(singleMessageMetadata.getPartitionKey()); } if (singleMessageMetadata.hasEventTime()) { msgMetadataBuilder.setEventTime(singleMessageMetadata.getEventTime()); } this.schema = schema; }
public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.MessageMetadata.Builder msgBuilder, ByteBuf payload, ByteBuf batchBuffer) { // build single message meta-data PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); if (msgBuilder.hasPartitionKey()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey()) .setPartitionKeyB64Encoded(msgBuilder.getPartitionKeyB64Encoded()); } if (!msgBuilder.getPropertiesList().isEmpty()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder .addAllProperties(msgBuilder.getPropertiesList()); } if (msgBuilder.hasEventTime()) { singleMessageMetadataBuilder.setEventTime(msgBuilder.getEventTime()); } try { return serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, payload, batchBuffer); } finally { singleMessageMetadataBuilder.recycle(); } }
public Builder toBuilder() { return newBuilder(this); }
public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized != -1) return isInitialized == 1; if (!hasPayloadSize()) { memoizedIsInitialized = 0; return false; } for (int i = 0; i < getPropertiesCount(); i++) { if (!getProperties(i).isInitialized()) { memoizedIsInitialized = 0; return false; } } memoizedIsInitialized = 1; return true; }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata prototype) {
public static Builder newBuilder(org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); }
public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseFrom( org.apache.pulsar.shaded.com.google.protobuf.v241.CodedInputStream input, org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { return newBuilder().mergeFrom(input, extensionRegistry) .buildParsed(); }
public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseFrom( java.io.InputStream input, org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { return newBuilder().mergeFrom(input, extensionRegistry) .buildParsed(); } public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseDelimitedFrom(java.io.InputStream input)
public Builder clearPartitionKey() { bitField0_ = (bitField0_ & ~0x00000002); partitionKey_ = getDefaultInstance().getPartitionKey(); return this; } void setPartitionKey(org.apache.pulsar.shaded.com.google.protobuf.v241.ByteString value) {
public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseFrom( byte[] data, org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) throws org.apache.pulsar.shaded.com.google.protobuf.v241.InvalidProtocolBufferException { return newBuilder().mergeFrom(data, extensionRegistry) .buildParsed(); } public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseFrom(java.io.InputStream input)
public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException { Builder builder = newBuilder(); if (builder.mergeDelimitedFrom(input)) { return builder.buildParsed(); } else { return null; } } public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseDelimitedFrom(
public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseDelimitedFrom( java.io.InputStream input, org.apache.pulsar.shaded.com.google.protobuf.v241.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { Builder builder = newBuilder(); if (builder.mergeDelimitedFrom(input, extensionRegistry)) { return builder.buildParsed(); } else { return null; } } public static org.apache.pulsar.common.api.proto.PulsarApi.SingleMessageMetadata parseFrom(