final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads); final ExecutorService executor = executorService(numThreads);
@Override public void init(SecorConfig config) throws UnknownHostException { this.mConfig = config; mConsumerConnector = Consumer.createJavaConsumerConnector(createConsumerConfig()); if (!mConfig.getKafkaTopicBlacklist().isEmpty() && !mConfig.getKafkaTopicFilter().isEmpty()) { throw new RuntimeException("Topic filter and blacklist cannot be both specified."); } TopicFilter topicFilter = !mConfig.getKafkaTopicBlacklist().isEmpty() ? new Blacklist(mConfig.getKafkaTopicBlacklist()) : new Whitelist(mConfig.getKafkaTopicFilter()); LOG.debug("Use TopicFilter {}({})", topicFilter.getClass(), topicFilter); List<KafkaStream<byte[], byte[]>> streams = mConsumerConnector.createMessageStreamsByFilter(topicFilter); KafkaStream<byte[], byte[]> stream = streams.get(0); mIterator = stream.iterator(); mKafkaMessageTimestampFactory = new KafkaMessageTimestampFactory(mConfig.getKafkaMessageTimestampClass()); }
private void init() { // register kafka offset lag metrics, one Gauge is for per consumer level granularity MetricRegistry registry = Metrics.getRegistry(); try { fetchedMsgCounter = registry.meter("kafkaIngesterConsumer." + this.getName() + "-msgFetchRate"); failedToIngestCounter = registry.meter("kafkaIngesterConsumer." + this.getName() + "-failedToIngest"); kafkaOffsetLagGauge = registry.register("kafkaIngesterConsumer." + this.getName() + "-kafkaOffsetLag", new JmxAttributeGauge( new ObjectName(maxLagMetricName), "Value")); } catch (MalformedObjectNameException | IllegalArgumentException e) { logger.error("Register failure for metrics of KafkaIngesterConsumer", e); } TopicFilter topicFilter = new Whitelist(AuditConfig.AUDIT_TOPIC_NAME); logger.info("{}: Topic filter is {}", getName(), AuditConfig.AUDIT_TOPIC_NAME); this.consumer = Consumer.createJavaConsumerConnector(createConsumerConfig()); KafkaStream<byte[], byte[]> stream = consumer.createMessageStreamsByFilter(topicFilter, 1).get(0); iterator = stream.iterator(); logger.info("KafkaIngesterConsumer thread {} is initialized successfully", getName()); if (AuditConfig.INGESTER_ENABLE_DEDUP) { deduplicator = new Deduplicator(threadId, AuditConfig.INGESTER_REDIS_HOST, AuditConfig.INGESTER_REDIS_PORT, AuditConfig.INGESTER_REDIS_KEY_TTL_SEC, AuditConfig.INGESTER_DUP_HOST_PREFIX, AuditConfig.INGESTER_HOSTS_WITH_DUP); deduplicator.open(); } else { deduplicator = null; } }
private List<KafkaStream<byte[], byte[]>> _createStreams(ConsumerConnector consumer, String topicName) { int numStreams = Math.max(Integer.parseInt( _configuration.getValue(Property.KAFKA_CONSUMER_STREAMS_PER_TOPIC.getName(), Property.KAFKA_CONSUMER_STREAMS_PER_TOPIC.getDefaultValue())), 2); return consumer.createMessageStreamsByFilter(new Whitelist(topicName), numStreams); }
private List<KafkaStream<byte[], byte[]>> _createStreams(ConsumerConnector consumer, String topicName) { int numStreams = Math.max(Integer.parseInt( _configuration.getValue(Property.KAFKA_CONSUMER_STREAMS_PER_TOPIC.getName(), Property.KAFKA_CONSUMER_STREAMS_PER_TOPIC.getDefaultValue())), 2); return consumer.createMessageStreamsByFilter(new Whitelist(topicName), numStreams); }
@Override public void startStream() { Properties props = new Properties(); props.setProperty("serializer.encoding", "UTF8"); ConsumerConfig consumerConfig = new ConsumerConfig(props); consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig); Whitelist topics = new Whitelist(config.getTopic()); VerifiableProperties vprops = new VerifiableProperties(props); inStreams = consumerConnector.createMessageStreamsByFilter(topics, 1, new StringDecoder(vprops), new StringDecoder(vprops)); for (final KafkaStream stream : inStreams) { executor.submit(new KafkaPersistReaderTask(this, stream)); } }
final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads); final ExecutorService executor = executorService(numThreads);
final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads); final ExecutorService executor = executorService(numThreads);
final List<KafkaStream<String, String>> streams = consumerConnector.createMessageStreamsByFilter( new Whitelist(Pattern.quote(topic)), 1, DEFAULT_STRING_DECODER, DEFAULT_STRING_DECODER );
@Override public void run() { ConsumerConfig cfg = new ConsumerConfig(props); consumer = Consumer.createJavaConsumerConnector(cfg); TopicFilter arg0 = new Whitelist(topic); List<KafkaStream<byte[], byte[]>> partitions = consumer.createMessageStreamsByFilter(arg0); while (!Thread.interrupted()) { for (KafkaStream<byte[], byte[]> partition : partitions) { ConsumerIterator<byte[], byte[]> it = partition .iterator(); while (it.hasNext()) { MessageAndMetadata<byte[], byte[]> msg = it.next(); onMessage(msg.topic(), new String(msg.message())); } } } } }, "consumer-" + topic);