ProducerInterceptors interceptors, Time time) { ProducerConfig config = new ProducerConfig(ProducerConfig.addSerializerToConfig(configs, keySerializer, valueSerializer)); try { Map<String, Object> userProvidedConfigs = config.originals(); this.producerConfig = config; this.time = time; String clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); if (clientId.length() <= 0) clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement(); MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS) .recordLevel(Sensor.RecordingLevel.forName(config.getString(ProducerConfig.METRICS_RECORDING_LEVEL_CONFIG))) .tags(metricTags); List<MetricsReporter> reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class, Collections.singletonMap(ProducerConfig.CLIENT_ID_CONFIG, clientId)); reporters.add(new JmxReporter(JMX_PREFIX)); this.metrics = new Metrics(metricConfig, reporters, time); this.partitioner = config.getConfiguredInstance(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioner.class); long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); if (keySerializer == null) { this.keySerializer = config.getConfiguredInstance(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Serializer.class); this.keySerializer.configure(config.originals(), true); } else { config.ignore(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
private static TransactionManager configureTransactionState(ProducerConfig config, LogContext logContext, Logger log) { TransactionManager transactionManager = null; boolean userConfiguredIdempotence = false; if (config.originals().containsKey(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG)) userConfiguredIdempotence = true; boolean userConfiguredTransactions = false; if (config.originals().containsKey(ProducerConfig.TRANSACTIONAL_ID_CONFIG)) userConfiguredTransactions = true; boolean idempotenceEnabled = config.getBoolean(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG); if (!idempotenceEnabled && userConfiguredIdempotence && userConfiguredTransactions) throw new ConfigException("Cannot set a " + ProducerConfig.TRANSACTIONAL_ID_CONFIG + " without also enabling idempotence."); if (userConfiguredTransactions) idempotenceEnabled = true; if (idempotenceEnabled) { String transactionalId = config.getString(ProducerConfig.TRANSACTIONAL_ID_CONFIG); int transactionTimeoutMs = config.getInt(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG); long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); transactionManager = new TransactionManager(logContext, transactionalId, transactionTimeoutMs, retryBackoffMs); if (transactionManager.isTransactional()) log.info("Instantiated a transactional producer."); else log.info("Instantiated an idempotent producer."); } return transactionManager; }
private static int configureDeliveryTimeout(ProducerConfig config, Logger log) { int deliveryTimeoutMs = config.getInt(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG); int lingerMs = config.getInt(ProducerConfig.LINGER_MS_CONFIG); int requestTimeoutMs = config.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); if (deliveryTimeoutMs < Integer.MAX_VALUE && deliveryTimeoutMs < lingerMs + requestTimeoutMs) { if (config.originals().containsKey(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG)) { // throw an exception if the user explicitly set an inconsistent value throw new ConfigException(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG + " should be equal to or larger than " + ProducerConfig.LINGER_MS_CONFIG + " + " + ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); } else { // override deliveryTimeoutMs default value to lingerMs + requestTimeoutMs for backward compatibility deliveryTimeoutMs = lingerMs + requestTimeoutMs; log.warn("{} should be equal to or larger than {} + {}. Setting it to {}.", ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, ProducerConfig.LINGER_MS_CONFIG, ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, deliveryTimeoutMs); } } return deliveryTimeoutMs; }
private static short configureAcks(ProducerConfig config, boolean idempotenceEnabled, Logger log) { boolean userConfiguredAcks = false; short acks = (short) parseAcks(config.getString(ProducerConfig.ACKS_CONFIG)); if (config.originals().containsKey(ProducerConfig.ACKS_CONFIG)) { userConfiguredAcks = true; } if (idempotenceEnabled && !userConfiguredAcks) { log.info("Overriding the default {} to all since idempotence is enabled.", ProducerConfig.ACKS_CONFIG); return -1; } if (idempotenceEnabled && acks != -1) { throw new ConfigException("Must set " + ProducerConfig.ACKS_CONFIG + " to all in order to use the idempotent " + "producer. Otherwise we cannot guarantee idempotence."); } return acks; }
this.producerConfig = config; this.time = new SystemTime(); MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG)) .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS); String clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG); if(clientId.length() <= 0) clientId = "producer-" + producerAutoId.getAndIncrement(); String jmxPrefix = "kafka.producer"; List<MetricsReporter> reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class); reporters.add(new JmxReporter(jmxPrefix)); this.metrics = new Metrics(metricConfig, reporters, time); this.partitioner = new Partitioner(); long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG); this.metadataFetchTimeoutMs = config.getLong(ProducerConfig.METADATA_FETCH_TIMEOUT_CONFIG); this.metadata = new Metadata(retryBackoffMs, config.getLong(ProducerConfig.METADATA_MAX_AGE_CONFIG)); this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG); this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG); this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG)); Map<String, String> metricTags = new LinkedHashMap<String, String>(); metricTags.put("client-id", clientId); this.accumulator = new RecordAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG), this.totalMemorySize, config.getLong(ProducerConfig.LINGER_MS_CONFIG), retryBackoffMs, config.getBoolean(ProducerConfig.BLOCK_ON_BUFFER_FULL_CONFIG), metrics, time,
final Map<String, Object> producerProps = getClientPropsWithPrefix(PRODUCER_PREFIX, ProducerConfig.configNames()); final int batchSize; if (producerProps.containsKey(ProducerConfig.BATCH_SIZE_CONFIG)) { batchSize = Integer.parseInt(producerProps.get(ProducerConfig.BATCH_SIZE_CONFIG).toString()); } else { final ProducerConfig producerDefaultConfig = new ProducerConfig(new Properties()); batchSize = producerDefaultConfig.getInt(ProducerConfig.BATCH_SIZE_CONFIG);
@Override public Properties overridingProps() { Properties props = new Properties(); int port = findLocalPort(); // We need to convert all the properties to the Cruise Control properties. setSecurityConfigs(props, "producer"); for (String configName : ProducerConfig.configNames()) { Object value = props.get(configName); if (value != null) { props.remove(configName); props.put(appendPrefix(configName), value); } } props.setProperty(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, CruiseControlMetricsReporter.class.getName()); props.setProperty(KafkaConfig.ListenersProp(), "SSL://127.0.0.1:" + port); props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG), "127.0.0.1:" + port); props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG), SecurityProtocol.SSL.name); props.setProperty(CRUISE_CONTROL_METRICS_REPORTING_INTERVAL_MS_CONFIG, "100"); props.setProperty(CRUISE_CONTROL_METRICS_TOPIC_CONFIG, TOPIC); props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), "1"); props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), "1"); return props; }
Sender newSender(LogContext logContext, KafkaClient kafkaClient, Metadata metadata) { int maxInflightRequests = configureInflightRequests(producerConfig, transactionManager != null); int requestTimeoutMs = producerConfig.getInt(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG); ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(producerConfig, time); ProducerMetrics metricsRegistry = new ProducerMetrics(this.metrics); Sensor throttleTimeSensor = Sender.throttleTimeSensor(metricsRegistry.senderMetrics); KafkaClient client = kafkaClient != null ? kafkaClient : new NetworkClient( new Selector(producerConfig.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), this.metrics, time, "producer", channelBuilder, logContext), metadata, clientId, maxInflightRequests, producerConfig.getLong(ProducerConfig.RECONNECT_BACKOFF_MS_CONFIG), producerConfig.getLong(ProducerConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG), producerConfig.getInt(ProducerConfig.SEND_BUFFER_CONFIG), producerConfig.getInt(ProducerConfig.RECEIVE_BUFFER_CONFIG), requestTimeoutMs, ClientDnsLookup.forConfig(producerConfig.getString(ProducerConfig.CLIENT_DNS_LOOKUP_CONFIG)), time, true, this.accumulator, maxInflightRequests == 1, producerConfig.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG), acks, retries, time, requestTimeoutMs, producerConfig.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG),
/** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings * are documented <a href="http://kafka.apache.org/documentation.html#producerconfigs">here</a>. Values can be * either strings or Objects of the appropriate type (for example a numeric configuration would accept either the * string "42" or the integer 42). * @param configs The producer configs * */ public KafkaProducer(Map<String, Object> configs) { this(new ProducerConfig(configs), null, null); }
private static int configureInflightRequests(ProducerConfig config, boolean idempotenceEnabled) { if (idempotenceEnabled && 5 < config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)) { throw new ConfigException("Must set " + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION + " to at most 5" + " to use the idempotent producer."); } return config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION); }
} catch (ClassCastException cce) { throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + " specified in key.serializer", cce); } catch (ClassCastException cce) { throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + " specified in value.serializer", cce);
/** * Merge boot producer properties, general properties from * {@link #setConfiguration(Map)} that apply to producers, properties from * {@link #setProducerProperties(Map)}, in that order. * @return the merged properties. */ public Map<String, Object> mergedProducerConfiguration() { Map<String, Object> producerConfiguration = new HashMap<>(); producerConfiguration.putAll(this.kafkaProperties.buildProducerProperties()); // Copy configured binder properties that apply to producers for (Map.Entry<String, String> configurationEntry : this.configuration.entrySet()) { if (ProducerConfig.configNames().contains(configurationEntry.getKey())) { producerConfiguration.put(configurationEntry.getKey(), configurationEntry.getValue()); } } producerConfiguration.putAll(this.producerProperties); // Override Spring Boot bootstrap server setting if left to default with the value // configured in the binder return getConfigurationWithBootstrapServer(producerConfiguration, ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); }
private static int configureRetries(ProducerConfig config, boolean idempotenceEnabled, Logger log) { boolean userConfiguredRetries = false; if (config.originals().containsKey(ProducerConfig.RETRIES_CONFIG)) { userConfiguredRetries = true; } if (idempotenceEnabled && !userConfiguredRetries) { // We recommend setting infinite retries when the idempotent producer is enabled, so it makes sense to make // this the default. log.info("Overriding the default retries config to the recommended value of {} since the idempotent " + "producer is enabled.", Integer.MAX_VALUE); return Integer.MAX_VALUE; } if (idempotenceEnabled && config.getInt(ProducerConfig.RETRIES_CONFIG) == 0) { throw new ConfigException("Must set " + ProducerConfig.RETRIES_CONFIG + " to non-zero when using the idempotent producer."); } return config.getInt(ProducerConfig.RETRIES_CONFIG); }
/** * A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings * are documented <a href="http://kafka.apache.org/documentation.html#producerconfigs">here</a>. * @param properties The producer configs */ public KafkaProducer(Properties properties) { this(new ProducerConfig(properties), null, null); }
} catch (ClassCastException cce) { throw new SerializationException("Can't convert key of class " + record.key().getClass().getName() + " to class " + producerConfig.getClass(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG).getName() + " specified in key.serializer"); } catch (ClassCastException cce) { throw new SerializationException("Can't convert value of class " + record.value().getClass().getName() + " to class " + producerConfig.getClass(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG).getName() + " specified in value.serializer");
/** * Merge boot producer properties, general properties from * {@link #setConfiguration(Map)} that apply to producers, properties from * {@link #setProducerProperties(Map)}, in that order. * @return the merged properties. */ public Map<String, Object> mergedProducerConfiguration() { Map<String, Object> producerConfiguration = new HashMap<>(); producerConfiguration.putAll(this.kafkaProperties.buildProducerProperties()); // Copy configured binder properties that apply to producers for (Map.Entry<String, String> configurationEntry : this.configuration.entrySet()) { if (ProducerConfig.configNames().contains(configurationEntry.getKey())) { producerConfiguration.put(configurationEntry.getKey(), configurationEntry.getValue()); } } producerConfiguration.putAll(this.producerProperties); // Override Spring Boot bootstrap server setting if left to default with the value // configured in the binder return getConfigurationWithBootstrapServer(producerConfiguration, ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); }
/** * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. * Valid configuration strings are documented <a href="http://kafka.apache.org/documentation.html#producerconfigs">here</a>. * @param properties The producer configs * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be * called when the serializer is passed in directly. * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't * be called when the serializer is passed in directly. */ public KafkaProducer(Properties properties, Serializer<K> keySerializer, Serializer<V> valueSerializer) { this(new ProducerConfig(addSerializerToConfig(properties, keySerializer, valueSerializer)), keySerializer, valueSerializer); }
/** * Get a map of custom configs by removing from the originals all the Streams, Consumer, Producer, and AdminClient configs. * Prefixed properties are also removed because they are already added by {@link #getClientPropsWithPrefix(String, Set)}. * This allows to set a custom property for a specific client alone if specified using a prefix, or for all * when no prefix is used. * * @return a map with the custom properties */ private Map<String, Object> getClientCustomProps() { final Map<String, Object> props = originals(); props.keySet().removeAll(CONFIG.names()); props.keySet().removeAll(ConsumerConfig.configNames()); props.keySet().removeAll(ProducerConfig.configNames()); props.keySet().removeAll(AdminClientConfig.configNames()); props.keySet().removeAll(originalsWithPrefix(CONSUMER_PREFIX, false).keySet()); props.keySet().removeAll(originalsWithPrefix(PRODUCER_PREFIX, false).keySet()); props.keySet().removeAll(originalsWithPrefix(ADMIN_CLIENT_PREFIX, false).keySet()); return props; }
/** * A producer is instantiated by providing a set of key-value pairs as configuration, a key and a value {@link Serializer}. * Valid configuration strings are documented <a href="http://kafka.apache.org/documentation.html#producerconfigs">here</a>. * Values can be either strings or Objects of the appropriate type (for example a numeric configuration would accept * either the string "42" or the integer 42). * @param configs The producer configs * @param keySerializer The serializer for key that implements {@link Serializer}. The configure() method won't be * called when the serializer is passed in directly. * @param valueSerializer The serializer for value that implements {@link Serializer}. The configure() method won't * be called when the serializer is passed in directly. */ public KafkaProducer(Map<String, Object> configs, Serializer<K> keySerializer, Serializer<V> valueSerializer) { this(new ProducerConfig(addSerializerToConfig(configs, keySerializer, valueSerializer)), keySerializer, valueSerializer); }
public Map<String, Object> getProducerConfiguration() { Map<String, Object> producerConfiguration = new HashMap<>(); // If Spring Boot Kafka properties are present, add them with lowest precedence if (this.kafkaProperties != null) { producerConfiguration.putAll(this.kafkaProperties.buildProducerProperties()); } // Copy configured binder properties for (Map.Entry<String, String> configurationEntry : configuration.entrySet()) { if (ProducerConfig.configNames().contains(configurationEntry.getKey())) { producerConfiguration.put(configurationEntry.getKey(), configurationEntry.getValue()); } } // Override Spring Boot bootstrap server setting if left to default with the value // configured in the binder if (ObjectUtils.isEmpty(producerConfiguration.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))) { producerConfiguration.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, getKafkaConnectionString()); } else { Object boostrapServersConfig = producerConfiguration.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); if (boostrapServersConfig instanceof List) { @SuppressWarnings("unchecked") List<String> bootStrapServers = (List<String>) producerConfiguration .get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); if (bootStrapServers.size() == 1 && bootStrapServers.get(0).equals("localhost:9092")) { producerConfiguration.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, getKafkaConnectionString()); } } } return Collections.unmodifiableMap(producerConfiguration); }