@Override public Map<String, String> toConfig() { Map<String, String> configs = new HashMap<>(super.toConfig()); if(!consumerZkConnect.isEmpty()) { configs.put(String.format(CONSUMER_ZK_CONNECT_CONFIG_KEY, getSystemName()), String.join(",", consumerZkConnect)); } consumerAutoOffsetResetOptional.ifPresent(consumerAutoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, getSystemName()), consumerAutoOffsetReset)); consumerFetchThresholdOptional.ifPresent(consumerFetchThreshold -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_CONFIG_KEY, getSystemName()), Integer.toString(consumerFetchThreshold))); consumerFetchThresholdBytesOptional.ifPresent(consumerFetchThresholdBytes -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_BYTES_CONFIG_KEY, getSystemName()), Long.toString(consumerFetchThresholdBytes))); consumerFetchMessageMaxBytesOptional.ifPresent(consumerFetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_KEY, getSystemName()), Long.toString(consumerFetchMessageMaxBytes))); consumerConfigs.forEach((key, value) -> configs.put(String.format(CONSUMER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); if (!producerBootstrapServers.isEmpty()) { configs.put(String.format(PRODUCER_BOOTSTRAP_SERVERS_CONFIG_KEY, getSystemName()), String.join(",", producerBootstrapServers)); } producerConfigs.forEach((key, value) -> configs.put(String.format(PRODUCER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); return configs; } }
@Test public void testSDConfigsWithOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka") .withConsumerZkConnect(ImmutableList.of("localhost:1234")) .withProducerBootstrapServers(ImmutableList.of("localhost:567", "localhost:890")) .withDefaultStreamOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST) .withConsumerAutoOffsetReset("smallest") .withConsumerFetchMessageMaxBytes(1024*1024) .withSamzaFetchThreshold(10000) .withSamzaFetchThresholdBytes(1024 * 1024) .withConsumerConfigs(ImmutableMap.of("custom-consumer-config-key", "custom-consumer-config-value")) .withProducerConfigs(ImmutableMap.of("custom-producer-config-key", "custom-producer-config-value")) .withDefaultStreamConfigs(ImmutableMap.of("custom-stream-config-key", "custom-stream-config-value")); Map<String, String> generatedConfigs = sd.toConfig(); assertEquals("org.apache.samza.system.kafka.KafkaSystemFactory", generatedConfigs.get("systems.kafka.samza.factory")); assertEquals("localhost:1234", generatedConfigs.get("systems.kafka.consumer.zookeeper.connect")); assertEquals("localhost:567,localhost:890", generatedConfigs.get("systems.kafka.producer.bootstrap.servers")); assertEquals("smallest", generatedConfigs.get("systems.kafka.consumer.auto.offset.reset")); assertEquals("1048576", generatedConfigs.get("systems.kafka.consumer.fetch.message.max.bytes")); assertEquals("10000", generatedConfigs.get("systems.kafka.samza.fetch.threshold")); assertEquals("1048576", generatedConfigs.get("systems.kafka.samza.fetch.threshold.bytes")); assertEquals("custom-consumer-config-value", generatedConfigs.get("systems.kafka.consumer.custom-consumer-config-key")); assertEquals("custom-producer-config-value", generatedConfigs.get("systems.kafka.producer.custom-producer-config-key")); assertEquals("custom-stream-config-value", generatedConfigs.get("systems.kafka.default.stream.custom-stream-config-key")); assertEquals("oldest", generatedConfigs.get("systems.kafka.default.stream.samza.offset.default")); assertEquals(11, generatedConfigs.size()); }
@Test public void testISDConfigsWithDefaults() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka") .withConsumerZkConnect(ImmutableList.of("localhost:123")) .withProducerBootstrapServers(ImmutableList.of("localhost:567", "localhost:890")); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())); Map<String, String> generatedConfigs = isd.toConfig(); assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals(1, generatedConfigs.size()); // verify that there are no other configs } }
@Test public void testSDConfigsWithoutOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka"); Map<String, String> generatedConfigs = sd.toConfig(); assertEquals("org.apache.samza.system.kafka.KafkaSystemFactory", generatedConfigs.get("systems.kafka.samza.factory")); assertEquals(1, generatedConfigs.size()); // verify that there are no other configs } }
@Test public void testISDConfigsWithOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka"); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())) .withConsumerAutoOffsetReset("largest") .withConsumerFetchMessageMaxBytes(1024 * 1024); Map<String, String> generatedConfigs = isd.toConfig();; assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("largest", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.auto.offset.reset")); assertEquals("1048576", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.fetch.message.max.bytes")); }
@Override public Map<String, String> toConfig() { Map<String, String> configs = new HashMap<>(super.toConfig()); if(!consumerZkConnect.isEmpty()) { configs.put(String.format(CONSUMER_ZK_CONNECT_CONFIG_KEY, getSystemName()), String.join(",", consumerZkConnect)); } consumerAutoOffsetResetOptional.ifPresent(consumerAutoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, getSystemName()), consumerAutoOffsetReset)); consumerFetchThresholdOptional.ifPresent(consumerFetchThreshold -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_CONFIG_KEY, getSystemName()), Integer.toString(consumerFetchThreshold))); consumerFetchThresholdBytesOptional.ifPresent(consumerFetchThresholdBytes -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_BYTES_CONFIG_KEY, getSystemName()), Long.toString(consumerFetchThresholdBytes))); consumerFetchMessageMaxBytesOptional.ifPresent(consumerFetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_KEY, getSystemName()), Long.toString(consumerFetchMessageMaxBytes))); consumerConfigs.forEach((key, value) -> configs.put(String.format(CONSUMER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); if (!producerBootstrapServers.isEmpty()) { configs.put(String.format(PRODUCER_BOOTSTRAP_SERVERS_CONFIG_KEY, getSystemName()), String.join(",", producerBootstrapServers)); } producerConfigs.forEach((key, value) -> configs.put(String.format(PRODUCER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); return configs; } }
@Override public Map<String, String> toConfig() { Map<String, String> configs = new HashMap<>(super.toConfig()); if(!consumerZkConnect.isEmpty()) { configs.put(String.format(CONSUMER_ZK_CONNECT_CONFIG_KEY, getSystemName()), String.join(",", consumerZkConnect)); } consumerAutoOffsetResetOptional.ifPresent(consumerAutoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, getSystemName()), consumerAutoOffsetReset)); consumerFetchThresholdOptional.ifPresent(consumerFetchThreshold -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_CONFIG_KEY, getSystemName()), Integer.toString(consumerFetchThreshold))); consumerFetchThresholdBytesOptional.ifPresent(consumerFetchThresholdBytes -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_BYTES_CONFIG_KEY, getSystemName()), Long.toString(consumerFetchThresholdBytes))); consumerFetchMessageMaxBytesOptional.ifPresent(consumerFetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_KEY, getSystemName()), Long.toString(consumerFetchMessageMaxBytes))); consumerConfigs.forEach((key, value) -> configs.put(String.format(CONSUMER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); if (!producerBootstrapServers.isEmpty()) { configs.put(String.format(PRODUCER_BOOTSTRAP_SERVERS_CONFIG_KEY, getSystemName()), String.join(",", producerBootstrapServers)); } producerConfigs.forEach((key, value) -> configs.put(String.format(PRODUCER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); return configs; } }