private static void receiveIndividualMessagesFromBatch(ReferenceCountedObject<MessageMetadata> msgMetadata, ByteBuf uncompressedPayload, long ledgerId, long entryId, MessageProcessor processor) { int batchSize = msgMetadata.get().getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } processor.process(RawMessageImpl.get(msgMetadata, singleMessageMetadataBuilder, singleMessagePayload, ledgerId, entryId, i)); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
public static MessageImpl<byte[]> deserialize(ByteBuf headersAndPayload) throws IOException { @SuppressWarnings("unchecked") MessageImpl<byte[]> msg = (MessageImpl<byte[]>) RECYCLER.get(); MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); msg.msgMetadataBuilder = MessageMetadata.newBuilder(msgMetadata); msgMetadata.recycle(); msg.payload = headersAndPayload; msg.messageId = null; msg.topic = null; msg.cnx = null; msg.properties = Collections.emptyMap(); return msg; }
public static ByteBuf newConnect(String authMethodName, String authData, String libVersion, String targetBroker, String originalPrincipal, String clientAuthData, String clientAuthMethod) { return newConnect(authMethodName, authData, getCurrentProtocolVersion(), libVersion, targetBroker, originalPrincipal, clientAuthData, clientAuthMethod); }
public static void skipChecksumIfPresent(ByteBuf buffer) { if (hasChecksum(buffer)) { readChecksum(buffer); } }
public static ByteBuf newConnected(int clientProtocolVersion) { CommandConnected.Builder connectedBuilder = CommandConnected.newBuilder(); connectedBuilder.setServerVersion("Pulsar Server"); // If the broker supports a newer version of the protocol, it will anyway advertise the max version that the // client supports, to avoid confusing the client. int currentProtocolVersion = getCurrentProtocolVersion(); int versionToAdvertise = Math.min(currentProtocolVersion, clientProtocolVersion); connectedBuilder.setProtocolVersion(versionToAdvertise); CommandConnected connected = connectedBuilder.build(); ByteBuf res = serializeWithSize(BaseCommand.newBuilder().setType(Type.CONNECTED).setConnected(connected)); connected.recycle(); connectedBuilder.recycle(); return res; }
MessageMetadata metadata = Commands.parseMessageMetadata(payload); ByteBuf batchBuffer = PooledByteBufAllocator.DEFAULT.buffer(payload.capacity()); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, 0, batchSize); if (!singleMessageMetadataBuilder.hasPartitionKey()) { messagesRetained++; Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, singleMessagePayload, batchBuffer); } else if (filter.test(singleMessageMetadataBuilder.getPartitionKey(), id) && singleMessagePayload.readableBytes() > 0) { messagesRetained++; Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, singleMessagePayload, batchBuffer); } else { Commands.serializeSingleMessageInBatchWithPayload(emptyMetadataBuilder, Unpooled.EMPTY_BUFFER, batchBuffer); MessageMetadata newMetadata = metadataBuilder.build(); ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, newMetadata, compressedPayload); Optional<RawMessage> result = Optional.of(new RawMessageImpl(msg.getMessageIdData(),
private TopicName validateTopicName(String topic, long requestId, GeneratedMessageLite requestCommand) { try { return TopicName.get(topic); } catch (Throwable t) { if (log.isDebugEnabled()) { log.debug("[{}] Failed to parse topic name '{}'", remoteAddress, topic, t); } if (requestCommand instanceof CommandLookupTopic) { ctx.writeAndFlush(Commands.newLookupErrorResponse(ServerError.InvalidTopicName, "Invalid topic name: " + t.getMessage(), requestId)); } else if (requestCommand instanceof CommandPartitionedTopicMetadata) { ctx.writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.InvalidTopicName, "Invalid topic name: " + t.getMessage(), requestId)); } else { ctx.writeAndFlush(Commands.newError(requestId, ServerError.InvalidTopicName, "Invalid topic name: " + t.getMessage())); } return null; } }
"[{}] Protocol version to advertise to broker is {}, clientProtocolVersion={}, proxyProtocolVersion={}", remoteAddress, protocolVersionToAdvertise, remoteEndpointProtocolVersion, Commands.getCurrentProtocolVersion()); ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, "Failed to authenticate")); close(); return; ctx.writeAndFlush(Commands.newConnected(protocolVersionToAdvertise));
private void performLookup(long clientRequestId, String topic, String brokerServiceUrl, boolean authoritative, int numberOfRetries) { if (numberOfRetries == 0) { proxyConnection.ctx().writeAndFlush(Commands.newLookupErrorResponse(ServerError.ServiceNotReady, "Reached max number of redirections", clientRequestId)); return; } catch (URISyntaxException e) { proxyConnection.ctx().writeAndFlush( Commands.newLookupErrorResponse(ServerError.MetadataError, e.getMessage(), clientRequestId)); return; command = Commands.newLookup(topic, authoritative, requestId); clientCnx.newLookup(command, requestId).thenAccept(result -> { String brokerUrl = connectWithTLS ? result.brokerUrlTls : result.brokerUrl; proxyConnection.ctx().writeAndFlush(Commands.newLookupResponse(brokerUrl, brokerUrl, true, LookupType.Connect, clientRequestId, true /* this is coming from proxy */)); log.warn("[{}] Failed to lookup topic {}: {}", clientAddress, topic, ex.getMessage()); proxyConnection.ctx().writeAndFlush( Commands.newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); return null; }); Commands.newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); return null; });
Commands.newPartitionMetadataResponse(metadata.partitions, clientRequestId)); }).exceptionally(ex -> { log.warn("[{}] Failed to get partitioned metadata for topic {} {}", clientAddress, topicName, ex.getMessage(), ex); proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse( ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); return null; brokerURI = new URI(brokerServiceURL); } catch (URISyntaxException e) { proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.MetadataError, e.getMessage(), clientRequestId)); return; command = Commands.newPartitionMetadataRequest(topicName.toString(), requestId); clientCnx.newLookup(command, requestId).thenAccept(lookupDataResult -> { proxyConnection.ctx().writeAndFlush( Commands.newPartitionMetadataResponse(lookupDataResult.partitions, clientRequestId)); }).exceptionally((ex) -> { log.warn("[{}] failed to get Partitioned metadata : {}", topicName.toString(), ex.getCause().getMessage(), ex); proxyConnection.ctx().writeAndFlush(Commands.newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); return null; proxyConnection.ctx().writeAndFlush(Commands.newPartitionMetadataResponse(ServerError.ServiceNotReady, ex.getMessage(), clientRequestId)); return null;
void doUnsubscribe(final long requestId) { final ChannelHandlerContext ctx = cnx.ctx(); subscription.doUnsubscribe(this).thenAccept(v -> { log.info("Unsubscribed successfully from {}", subscription); cnx.removedConsumer(this); ctx.writeAndFlush(Commands.newSuccess(requestId)); }).exceptionally(exception -> { log.warn("Unsubscribe failed for {}", subscription, exception); ctx.writeAndFlush( Commands.newError(requestId, BrokerServiceException.getClientErrorCode(exception.getCause()), exception.getCause().getMessage())); return null; }); }
public static List<ImmutablePair<MessageId,String>> extractIdsAndKeys(RawMessage msg) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); int batchSize = metadata.getNumMessagesInBatch(); CompressionType compressionType = metadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = metadata.getUncompressedSize(); ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize); metadata.recycle(); List<ImmutablePair<MessageId,String>> idsAndKeys = new ArrayList<>(); for (int i = 0; i < batchSize; i++) { SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, 0, batchSize); MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); if (!singleMessageMetadataBuilder.getCompactedOut()) { idsAndKeys.add(ImmutablePair.of(id, singleMessageMetadataBuilder.getPartitionKey())); } singleMessageMetadataBuilder.recycle(); singleMessagePayload.release(); } uncompressedPayload.release(); return idsAndKeys; }
CommandGetTopicsOfNamespace.Mode mode) { if (numberOfRetries == 0) { proxyConnection.ctx().writeAndFlush(Commands.newError(clientRequestId, ServerError.ServiceNotReady, "Reached max number of redirections")); return; } catch (URISyntaxException e) { proxyConnection.ctx().writeAndFlush( Commands.newError(clientRequestId, ServerError.MetadataError, e.getMessage())); return; command = Commands.newGetTopicsOfNamespaceRequest(namespaceName, requestId, mode); clientCnx.newGetTopicsOfNamespace(command, requestId).thenAccept(topicList -> proxyConnection.ctx().writeAndFlush( Commands.newGetTopicsOfNamespaceResponse(topicList, clientRequestId)) ).exceptionally(ex -> { log.warn("[{}] Failed to get TopicsOfNamespace {}: {}", clientAddress, namespaceName, ex.getMessage()); proxyConnection.ctx().writeAndFlush( Commands.newError(clientRequestId, ServerError.ServiceNotReady, ex.getMessage())); return null; }); Commands.newError(clientRequestId, ServerError.ServiceNotReady, ex.getMessage())); return null; });
private void sendLookupResponse(long requestId) { try { LoadManagerReport availableBroker = service.getDiscoveryProvider().nextBroker(); ctx.writeAndFlush(Commands.newLookupResponse(availableBroker.getPulsarServiceUrl(), availableBroker.getPulsarServiceUrlTls(), false, Redirect, requestId, false)); } catch (PulsarServerException e) { LOG.warn("[{}] Failed to get next active broker {}", remoteAddress, e.getMessage(), e); ctx.writeAndFlush( Commands.newLookupErrorResponse(ServerError.ServiceNotReady, e.getMessage(), requestId)); } }
String msg = "Unable to authenticate"; LOG.warn("[{}] {}: {}", remoteAddress, msg, e.getMessage()); ctx.writeAndFlush(Commands.newError(-1, ServerError.AuthenticationError, msg)); close(); return; ctx.writeAndFlush(Commands.newConnected(connect.getProtocolVersion())); state = State.Connected; remoteEndpointProtocolVersion = connect.getProtocolVersion();
@Override protected void handleGetTopicsOfNamespace(CommandGetTopicsOfNamespace commandGetTopicsOfNamespace) { final long requestId = commandGetTopicsOfNamespace.getRequestId(); final String namespace = commandGetTopicsOfNamespace.getNamespace(); final CommandGetTopicsOfNamespace.Mode mode = commandGetTopicsOfNamespace.getMode(); try { final NamespaceName namespaceName = NamespaceName.get(namespace); final List<String> topics = getBrokerService().pulsar().getNamespaceService() .getListOfTopics(namespaceName, mode); if (log.isDebugEnabled()) { log.debug("[{}] Received CommandGetTopicsOfNamespace for namespace [//{}] by {}, size:{}", remoteAddress, namespace, requestId, topics.size()); } ctx.writeAndFlush(Commands.newGetTopicsOfNamespaceResponse(topics, requestId)); } catch (Exception e) { log.warn("[{]] Error GetTopicsOfNamespace for namespace [//{}] by {}", remoteAddress, namespace, requestId); ctx.writeAndFlush( Commands.newError(requestId, BrokerServiceException.getClientErrorCode(new ServerMetadataException(e)), e.getMessage())); } }
protected ByteBuf newConnectCommand() throws PulsarClientException { String authData = ""; if (authentication.getAuthData().hasDataFromCommand()) { authData = authentication.getAuthData().getCommandData(); } return Commands.newConnect(authentication.getAuthMethodName(), authData, this.protocolVersion, getPulsarClientVersion(), proxyToTargetBrokerAddress, null, null, null); }
public void publishMessage(long producerId, long sequenceId, ByteBuf headersAndPayload, long batchSize) { if (isClosed) { cnx.ctx().channel().eventLoop().execute(() -> { cnx.ctx().writeAndFlush(Commands.newSendError(producerId, sequenceId, ServerError.PersistenceError, "Producer is closed")); cnx.completedSendOperation(isNonPersistentTopic); cnx.ctx().channel().eventLoop().execute(() -> { cnx.ctx().writeAndFlush( Commands.newSendError(producerId, sequenceId, ServerError.ChecksumError, "Checksum failed on the broker")); cnx.completedSendOperation(isNonPersistentTopic); }); MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload); headersAndPayload.resetReaderIndex(); log.warn("[{}] Messages must be encrypted", getTopic().getName()); cnx.ctx().channel().eventLoop().execute(() -> { cnx.ctx().writeAndFlush(Commands.newSendError(producerId, sequenceId, ServerError.MetadataError, "Messages must be encrypted")); cnx.completedSendOperation(isNonPersistentTopic);
log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName); ctx.writeAndFlush(Commands.newError(requestId, ServerError.AuthorizationError, msg)); return; log.info("[{}] Producer with the same id is already created: {}", remoteAddress, producer); ctx.writeAndFlush(Commands.newProducerSuccess(requestId, producer.getProducerName(), producer.getSchemaVersion())); return null; log.warn("[{}][{}] Producer is already present on the connection", remoteAddress, topicName); ctx.writeAndFlush(Commands.newError(requestId, error, "Producer is already present on the connection")); return null; if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_request_hold) { ctx.writeAndFlush( Commands.newError(requestId, ServerError.ProducerBlockedQuotaExceededError, illegalStateException.getMessage())); } else if (retentionPolicy == BacklogQuota.RetentionPolicy.producer_exception) { ctx.writeAndFlush(Commands.newError(requestId, ServerError.ProducerBlockedQuotaExceededException, illegalStateException.getMessage())); String msg = String.format("Encryption is required in %s", topicName); log.warn("[{}] {}", remoteAddress, msg); ctx.writeAndFlush(Commands.newError(requestId, ServerError.MetadataError, msg));
public ClientCnx(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) { this(conf, eventLoopGroup, Commands.getCurrentProtocolVersion()); }