final int numMessages = msgMetadata.getNumMessagesInBatch(); if (isMessageUndecryptable || (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch())) { final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), msgId, msgMetadata, uncompressedPayload, createEncryptionContext(msgMetadata), cnx, schema, redeliveryCount); uncompressedPayload.release(); msgMetadata.recycle(); msgMetadata.recycle();
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata.getDefaultInstance()) return this; if (other.hasProducerName()) { setProducerName(other.getProducerName()); if (other.hasSequenceId()) { setSequenceId(other.getSequenceId()); if (other.hasPublishTime()) { setPublishTime(other.getPublishTime()); if (other.hasReplicatedFrom()) { setReplicatedFrom(other.getReplicatedFrom()); if (other.hasPartitionKey()) { setPartitionKey(other.getPartitionKey()); if (other.hasCompression()) { setCompression(other.getCompression()); if (other.hasUncompressedSize()) { setUncompressedSize(other.getUncompressedSize()); if (other.hasNumMessagesInBatch()) { setNumMessagesInBatch(other.getNumMessagesInBatch()); if (other.hasEventTime()) { setEventTime(other.getEventTime()); if (other.hasEncryptionAlgo()) {
private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, ClientCnx currentCnx) { if (msgMetadata.getEncryptionKeysCount() == 0) { return payload.retain();
final int numMessages = msgMetadata.getNumMessagesInBatch(); if (isMessageUndecryptable || (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch())) { final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), msgId, msgMetadata, uncompressedPayload, createEncryptionContext(msgMetadata), cnx, schema); uncompressedPayload.release(); msgMetadata.recycle(); msgMetadata.recycle();
ByteBufPair cmd = sendMessage(producerId, sequenceId, 1, msgMetadata, encryptedPayload); msgMetadataBuilder.recycle(); msgMetadata.recycle();
void receiveIndividualMessagesFromBatch(MessageMetadata msgMetadata, int redeliveryCount, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx) { int batchSize = msgMetadata.getNumMessagesInBatch();
for (KeyValue keyValue : metadata.getPropertiesList()) { responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue()); if (metadata.hasPublishTime()) { responseBuilder.header("X-Pulsar-publish-time", DateFormatter.format(metadata.getPublishTime())); if (metadata.hasEventTime()) { responseBuilder.header("X-Pulsar-event-time", DateFormatter.format(metadata.getEventTime())); if (metadata.hasNumMessagesInBatch()) { responseBuilder.header("X-Pulsar-num-batch-message", metadata.getNumMessagesInBatch()); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression()); ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
ByteBuf batchBuffer = PooledByteBufAllocator.DEFAULT.buffer(payload.capacity()); CompressionType compressionType = metadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); try { int batchSize = metadata.getNumMessagesInBatch(); int messagesRetained = 0; ByteBuf compressedPayload = codec.encode(batchBuffer); MessageMetadata.Builder metadataBuilder = metadata.toBuilder(); metadataBuilder.setUncompressedSize(newUncompressedSize); MessageMetadata newMetadata = metadataBuilder.build(); metadataAndPayload)); metadataBuilder.recycle(); newMetadata.recycle(); metadataAndPayload.release(); compressedPayload.release(); metadata.recycle(); msg.close();
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata.getDefaultInstance()) return this; if (other.hasProducerName()) { setProducerName(other.getProducerName()); if (other.hasSequenceId()) { setSequenceId(other.getSequenceId()); if (other.hasPublishTime()) { setPublishTime(other.getPublishTime()); if (other.hasReplicatedFrom()) { setReplicatedFrom(other.getReplicatedFrom()); if (other.hasPartitionKey()) { setPartitionKey(other.getPartitionKey()); if (other.hasCompression()) { setCompression(other.getCompression()); if (other.hasUncompressedSize()) { setUncompressedSize(other.getUncompressedSize()); if (other.hasNumMessagesInBatch()) { setNumMessagesInBatch(other.getNumMessagesInBatch()); if (other.hasEventTime()) { setEventTime(other.getEventTime()); if (other.hasEncryptionAlgo()) {
if (msgMetadata.getEncryptionKeysCount() > 0) { throw new IOException("Cannot parse encrypted message " + msgMetadata + " on topic " + topicName); final int numMessages = msgMetadata.getNumMessagesInBatch(); if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) { final MessageImpl<?> message = new MessageImpl<>(topicName.toString(), msgId, msgMetadata, uncompressedPayload, msgMetadata.recycle();
private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, ClientCnx currentCnx) { if (msgMetadata.getEncryptionKeysCount() == 0) { return payload.retain();
producerName = md.getProducerName(); sequenceId = md.getSequenceId(); publishContext.setOriginalProducerName(producerName); publishContext.setOriginalSequenceId(sequenceId); headersAndPayload.readerIndex(readerIndex); md.recycle();
if (msgMetadata.getEncryptionKeysCount() < 1) { log.warn("[{}] Messages must be encrypted", getTopic().getName()); cnx.ctx().channel().eventLoop().execute(() -> {
public static void receiveIndividualMessagesFromBatch(String topic, MessageMetadata msgMetadata, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, int partitionIndex, MessageProcessor processor) { int batchSize = msgMetadata.getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex, i, null); final MessageImpl<?> message = new MessageImpl<>( topic, batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, Optional.empty(), cnx, null); processor.process(batchMessageIdImpl, message, singleMessagePayload); singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
if (isInitialized != -1) return isInitialized == 1; if (!hasProducerName()) { memoizedIsInitialized = 0; return false; if (!hasSequenceId()) { memoizedIsInitialized = 0; return false; if (!hasPublishTime()) { memoizedIsInitialized = 0; return false; for (int i = 0; i < getPropertiesCount(); i++) { if (!getProperties(i).isInitialized()) { memoizedIsInitialized = 0; return false; for (int i = 0; i < getEncryptionKeysCount(); i++) { if (!getEncryptionKeys(i).isInitialized()) { memoizedIsInitialized = 0; return false;
MessageImpl(String topic, BatchMessageIdImpl batchMessageIdImpl, MessageMetadata msgMetadata, PulsarApi.SingleMessageMetadata singleMessageMetadata, ByteBuf payload, Optional<EncryptionContext> encryptionCtx, ClientCnx cnx, Schema<T> schema) { this.msgMetadataBuilder = MessageMetadata.newBuilder(msgMetadata); this.messageId = batchMessageIdImpl; this.topic = topic; this.cnx = cnx; this.payload = Unpooled.copiedBuffer(payload); this.encryptionCtx = encryptionCtx; if (singleMessageMetadata.getPropertiesCount() > 0) { Map<String, String> properties = Maps.newTreeMap(); for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } this.properties = Collections.unmodifiableMap(properties); } else { properties = Collections.emptyMap(); } if (singleMessageMetadata.hasPartitionKey()) { msgMetadataBuilder.setPartitionKeyB64Encoded(singleMessageMetadata.getPartitionKeyB64Encoded()); msgMetadataBuilder.setPartitionKey(singleMessageMetadata.getPartitionKey()); } if (singleMessageMetadata.hasEventTime()) { msgMetadataBuilder.setEventTime(singleMessageMetadata.getEventTime()); } this.schema = schema; }
public static List<ImmutablePair<MessageId,String>> extractIdsAndKeys(RawMessage msg) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); int batchSize = metadata.getNumMessagesInBatch(); CompressionType compressionType = metadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); metadata.recycle(); List<ImmutablePair<MessageId,String>> idsAndKeys = new ArrayList<>(); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, 0, batchSize); MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); if (!singleMessageMetadataBuilder.getCompactedOut()) { idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey())); } singleMessageMetadataBuilder.recycle(); singleMessagePayload.release(); } uncompressedPayload.release(); return idsAndKeys; }
if (msgMetadata.getEncryptionKeysCount() > 0) { encryptionCtx = new EncryptionContext(); Map<String, EncryptionKey> keys = msgMetadata.getEncryptionKeysList().stream() .collect( Collectors.toMap(EncryptionKeys::getKey, : null))); byte[] encParam = new byte[MessageCrypto.ivLen]; msgMetadata.getEncryptionParam().copyTo(encParam, 0); Optional<Integer> batchSize = Optional .ofNullable(msgMetadata.hasNumMessagesInBatch() ? msgMetadata.getNumMessagesInBatch() : null); encryptionCtx.setKeys(keys); encryptionCtx.setParam(encParam); encryptionCtx.setAlgorithm(msgMetadata.getEncryptionAlgo()); encryptionCtx.setCompressionType(msgMetadata.getCompression()); encryptionCtx.setUncompressedMessageSize(msgMetadata.getUncompressedSize()); encryptionCtx.setBatchSize(batchSize);
private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, ClientCnx currentCnx) { CompressionType compressionType = msgMetadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = msgMetadata.getUncompressedSize(); int payloadSize = payload.readableBytes(); if (payloadSize > PulsarDecoder.MaxMessageSize) { // payload size is itself corrupted since it cannot be bigger than the MaxMessageSize log.error("[{}][{}] Got corrupted payload message size {} at {}", topic, subscription, payloadSize, messageId); discardCorruptedMessage(messageId, currentCnx, ValidationError.UncompressedSizeCorruption); return null; } try { ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); return uncompressedPayload; } catch (IOException e) { log.error("[{}][{}] Failed to decompress message with {} at {}: {}", topic, subscription, compressionType, messageId, e.getMessage(), e); discardCorruptedMessage(messageId, currentCnx, ValidationError.DecompressionError); return null; } }
private ByteBuf decryptData(SecretKey dataKeySecret, MessageMetadata msgMetadata, ByteBuf payload) { // unpack iv and encrypted data ByteString ivString = msgMetadata.getEncryptionParam(); ivString.copyTo(iv, 0); GCMParameterSpec gcmParams = new GCMParameterSpec(tagLen, iv); ByteBuf targetBuf = null; try { cipher.init(Cipher.DECRYPT_MODE, dataKeySecret, gcmParams); ByteBuffer sourceNioBuf = payload.nioBuffer(payload.readerIndex(), payload.readableBytes()); int maxLength = cipher.getOutputSize(payload.readableBytes()); targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength); ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength); int decryptedSize = cipher.doFinal(sourceNioBuf, targetNioBuf); targetBuf.writerIndex(decryptedSize); } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException | BadPaddingException | ShortBufferException e) { log.error("{} Failed to decrypt message {}", logCtx, e.getMessage()); if (targetBuf != null) { targetBuf.release(); targetBuf = null; } } return targetBuf; }