public static Map<String, Object> getKafkaParams(@NonNull final KafkaConfiguration kafkaConf) { final Map<String, Object> newKafkaParams = new HashMap<>(); kafkaConf.getKafkaParams().entrySet().stream().forEach( entry -> { final String val = entry.getValue(); try { final long longVal = Long.parseLong(val); if (longVal >= Integer.MAX_VALUE || longVal <= Integer.MIN_VALUE) { newKafkaParams.put(entry.getKey(), longVal); } else { newKafkaParams.put(entry.getKey(), (int) longVal); } return; } catch (NumberFormatException e) { // ignore it. } // Add all remaining (key,value) pairs as Strings. newKafkaParams.put(entry.getKey(), entry.getValue()); } ); return newKafkaParams; } }
public KafkaConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, getMandatoryProperties()); this.kafkaParams = new HashMap<>(); this.kafkaParams.put(GROUP_ID, DEFAULT_GROUP_ID); this.kafkaParams.putAll(getConf().getPropertiesWithPrefix(KAFKA_CONNECTION_PREFIX, true)); this.kafkaParams.put(KEY_DESERIALIZER, ByteArrayDeserializer.class.getCanonicalName()); this.kafkaParams.put(VALUE_DESERIALIZER, ByteArrayDeserializer.class.getCanonicalName()); this.kafkaParams.put(ENABLE_AUTO_COMMIT, ENABLE_AUTO_COMMIT_VALUE); // If retry backoff is not set then we would want to reduce it to lower values. Default value is 400ms. if (!kafkaParams.containsKey(KAFKA_BACKOFF_MS_CONFIG)) { kafkaParams.put(KAFKA_BACKOFF_MS_CONFIG, DEFAULT_KAFKA_BACKOFF_MS_CONFIG); } }
@Test public void testGroupIdDefault() { final Configuration conf = new Configuration(); KafkaTestHelper.setMandatoryConf(conf, Collections.singletonList(KAFKA_BROKER_LIST), Collections.singletonList(BROKER_LIST)); final KafkaConfiguration kafkaConf = new KafkaConfiguration(conf); Assert.assertEquals(DEFAULT_GROUP_ID, kafkaConf.getKafkaParams().get(GROUP_ID)); }
public List<String> getMandatoryProperties() { final List<String> ret = new LinkedList<>(); ret.addAll(super.getMandatoryProperties()); ret.add(KAFKA_TOPIC_NAME); ret.add(KAFKA_CLUSTER_NAME); ret.add(KAFKA_START_DATE); return ret; }
@Test public void testGroupIdConfigurable() { final String myGroupId = "mygroup"; final Configuration conf = new Configuration(); KafkaTestHelper.setMandatoryConf(conf, Arrays.asList(KAFKA_BROKER_LIST, KAFKA_GROUP_ID), Arrays.asList(BROKER_LIST, myGroupId)); final KafkaConfiguration kafkaConf = new KafkaConfiguration(conf); Assert.assertEquals(myGroupId, kafkaConf.getKafkaParams().get(GROUP_ID)); }
@Test public void testAutoCommitNotConfigurable() { final Configuration conf = new Configuration(); KafkaTestHelper.setMandatoryConf(conf, Arrays.asList(KAFKA_BROKER_LIST, KAFKA_PROPERTY_PREFIX + ENABLE_AUTO_COMMIT), Arrays.asList(BROKER_LIST, "bar")); final KafkaConfiguration kafkaConf = new KafkaConfiguration(conf); Assert.assertEquals(ENABLE_AUTO_COMMIT_VALUE, kafkaConf.getKafkaParams().get(ENABLE_AUTO_COMMIT)); }