protected PublisherPool createPublisherPool(final ProcessContext context) { final int maxMessageSize = context.getProperty(MAX_REQUEST_SIZE).asDataSize(DataUnit.B).intValue(); final long maxAckWaitMillis = context.getProperty(ACK_WAIT_TIME).asTimePeriod(TimeUnit.MILLISECONDS).longValue(); final Map<String, Object> kafkaProperties = new HashMap<>(); KafkaProcessorUtils.buildCommonKafkaProperties(context, ProducerConfig.class, kafkaProperties); kafkaProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); kafkaProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); kafkaProperties.put("max.request.size", String.valueOf(maxMessageSize)); return new PublisherPool(kafkaProperties, getLogger(), maxMessageSize, maxAckWaitMillis); }
private synchronized PublisherPool getPublisherPool(final ProcessContext context) { PublisherPool pool = publisherPool; if (pool != null) { return pool; } return publisherPool = createPublisherPool(context); }
final PublisherPool pool = getPublisherPool(context); if (pool == null) { context.yield(); if (!isScheduled()) { final byte[] messageKey = getMessageKey(flowFile, context); final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue(); final byte[] demarcatorBytes; final int successCount = publishResult.getSuccessfulMessageCount(failure); if (successCount > 0) { getLogger().error("Failed to send some messages for {} to Kafka, but {} messages were acknowledged by Kafka. Routing to failure due to {}", new Object[] {failure, successCount, publishResult.getReasonForFailure(failure)}); } else { getLogger().error("Failed to send all message for {} to Kafka; routing to failure due to {}", new Object[] {failure, publishResult.getReasonForFailure(failure)});
/** * Will rendezvous with Kafka if {@link ProcessSession} contains {@link FlowFile} * producing a result {@link FlowFile}. * <br> * The result {@link FlowFile} that is successful is then transfered to {@link #REL_SUCCESS} * <br> * The result {@link FlowFile} that is failed is then transfered to {@link #REL_FAILURE} * */ @Override protected boolean rendezvousWithKafka(ProcessContext context, ProcessSession session){ FlowFile flowFile = session.get(); if (flowFile != null) { long start = System.nanoTime(); flowFile = this.doRendezvousWithKafka(flowFile, context, session); Relationship relationship = REL_SUCCESS; if (!this.isFailedFlowFile(flowFile)) { String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue(); long executionDuration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); String transitUri = this.buildTransitURI(context.getProperty(SECURITY_PROTOCOL).getValue(), this.brokers, topic); session.getProvenanceReporter().send(flowFile, transitUri, "Sent " + flowFile.getAttribute(MSG_COUNT) + " Kafka messages", executionDuration); this.getLogger().info("Successfully sent {} to Kafka as {} message(s) in {} millis", new Object[] { flowFile, flowFile.getAttribute(MSG_COUNT), executionDuration }); } else { relationship = REL_FAILURE; flowFile = session.penalize(flowFile); } session.transfer(flowFile, relationship); } return flowFile != null; }
/** * Builds and instance of {@link KafkaPublisher}. */ @Override protected KafkaPublisher buildKafkaResource(ProcessContext context, ProcessSession session) { Properties kafkaProperties = this.buildKafkaProperties(context); kafkaProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); kafkaProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); this.brokers = context.getProperty(BOOTSTRAP_SERVERS).evaluateAttributeExpressions().getValue(); KafkaPublisher publisher = new KafkaPublisher(kafkaProperties, this.getLogger()); return publisher; }
/** * Will rendezvous with {@link KafkaPublisher} after building * {@link PublishingContext} and will produce the resulting {@link FlowFile}. * The resulting FlowFile contains all required information to determine * if message publishing originated from the provided FlowFile has actually * succeeded fully, partially or failed completely (see * {@link #isFailedFlowFile(FlowFile)}. */ private FlowFile doRendezvousWithKafka(final FlowFile flowFile, final ProcessContext context, final ProcessSession session) { final AtomicReference<KafkaPublisherResult> publishResultRef = new AtomicReference<>(); session.read(flowFile, new InputStreamCallback() { @Override public void process(InputStream contentStream) throws IOException { PublishingContext publishingContext = PublishKafka.this.buildPublishingContext(flowFile, context, contentStream); KafkaPublisherResult result = PublishKafka.this.kafkaResource.publish(publishingContext); publishResultRef.set(result); } }); FlowFile resultFile = publishResultRef.get().isAllAcked() ? this.cleanUpFlowFileIfNecessary(flowFile, session) : session.putAllAttributes(flowFile, this.buildFailedFlowFileAttributes(publishResultRef.get().getLastMessageAcked(), flowFile, context)); if (!this.isFailedFlowFile(resultFile)) { resultFile = session.putAttribute(resultFile, MSG_COUNT, String.valueOf(publishResultRef.get().getMessagesSent())); } return resultFile; }
/** * Returns 'true' if provided FlowFile is a failed FlowFile. A failed * FlowFile contains {@link #FAILED_PROC_ID_ATTR}. */ private boolean isFailedFlowFile(FlowFile flowFile) { return this.getIdentifier().equals(flowFile.getAttribute(FAILED_PROC_ID_ATTR)); } }
final PublisherPool pool = getPublisherPool(context); if (pool == null) { context.yield(); if (!isScheduled()) { final byte[] messageKey = getMessageKey(flowFile, context); final String topic = context.getProperty(TOPIC).evaluateAttributeExpressions(flowFile).getValue(); final byte[] demarcatorBytes; final int successCount = publishResult.getSuccessfulMessageCount(failure); if (successCount > 0) { getLogger().error("Failed to send some messages for {} to Kafka, but {} messages were acknowledged by Kafka. Routing to failure due to {}", new Object[] {failure, successCount, publishResult.getReasonForFailure(failure)}); } else { getLogger().error("Failed to send all message for {} to Kafka; routing to failure due to {}", new Object[] {failure, publishResult.getReasonForFailure(failure)});
/** * Builds a {@link Map} of FAILED_* attributes * * @see #FAILED_PROC_ID_ATTR * @see #FAILED_LAST_ACK_IDX * @see #FAILED_TOPIC_ATTR * @see #FAILED_KEY_ATTR * @see #FAILED_DELIMITER_ATTR */ private Map<String, String> buildFailedFlowFileAttributes(int lastAckedMessageIndex, FlowFile sourceFlowFile, ProcessContext context) { Map<String, String> attributes = new HashMap<>(); attributes.put(FAILED_PROC_ID_ATTR, this.getIdentifier()); attributes.put(FAILED_LAST_ACK_IDX, String.valueOf(lastAckedMessageIndex)); attributes.put(FAILED_TOPIC_ATTR, context.getProperty(TOPIC).evaluateAttributeExpressions(sourceFlowFile).getValue()); attributes.put(FAILED_KEY_ATTR, context.getProperty(KEY).evaluateAttributeExpressions(sourceFlowFile).getValue()); attributes.put(FAILED_DELIMITER_ATTR, context.getProperty(MESSAGE_DEMARCATOR).isSet() ? context.getProperty(MESSAGE_DEMARCATOR).evaluateAttributeExpressions(sourceFlowFile).getValue() : null); return attributes; }
protected PublisherPool createPublisherPool(final ProcessContext context) { final int maxMessageSize = context.getProperty(MAX_REQUEST_SIZE).asDataSize(DataUnit.B).intValue(); final long maxAckWaitMillis = context.getProperty(ACK_WAIT_TIME).asTimePeriod(TimeUnit.MILLISECONDS).longValue(); final Map<String, Object> kafkaProperties = new HashMap<>(); KafkaProcessorUtils.buildCommonKafkaProperties(context, ProducerConfig.class, kafkaProperties); kafkaProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); kafkaProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); kafkaProperties.put("max.request.size", String.valueOf(maxMessageSize)); return new PublisherPool(kafkaProperties, getLogger(), maxMessageSize, maxAckWaitMillis); }
private synchronized PublisherPool getPublisherPool(final ProcessContext context) { PublisherPool pool = publisherPool; if (pool != null) { return pool; } return publisherPool = createPublisherPool(context); }