proxy.addTopicPartition(topicPartitionsToSSP.get(tp), startingOffset); }); if (proxy != null && !proxy.isRunning()) { LOG.info("{}: Starting proxy {}", this, proxy); proxy.start();
KafkaConsumerProxy(Consumer<K, V> kafkaConsumer, String systemName, String clientId, KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, String metricName) { this.kafkaConsumer = kafkaConsumer; this.systemName = systemName; this.sink = messageSink; this.kafkaConsumerMetrics = samzaConsumerMetrics; this.metricName = metricName; this.clientId = clientId; this.kafkaConsumerMetrics.registerClientProxy(metricName); consumerPollThread = new Thread(createProxyThreadRunnable()); consumerPollThread.setDaemon(true); consumerPollThread.setName( "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); LOG.info("Creating KafkaConsumerProxy with systeName={}, clientId={}, metricsName={}", systemName, clientId, metricName); }
private Runnable createProxyThreadRunnable() { Runnable runnable = () -> { isRunning = true; try { consumerPollThreadStartLatch.countDown(); LOG.info("Starting consumer poll thread {} for system {}", consumerPollThread.getName(), systemName); initializeLags(); while (isRunning) { fetchMessages(); } } catch (Throwable throwable) { LOG.error(String.format("Error in KafkaConsumerProxy poll thread for system: %s.", systemName), throwable); // KafkaSystemConsumer uses the failureCause to propagate the throwable to the container failureCause = throwable; isRunning = false; } if (!isRunning) { LOG.info("KafkaConsumerProxy for system {} has stopped.", systemName); } }; return runnable; }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { LOG.info("{}: KafkaConsumerProxy is not running. Stopping the consumer.", this); stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
response = pollConsumer(sspsToFetch, 500L); List<IncomingMessageEnvelope> envelopes = e.getValue(); if (envelopes != null) { moveMessagesToTheirQueue(e.getKey(), envelopes); populateCurrentLags(sspsToFetch); // find current lags for for each SSP } else { // nothing to read refreshLagMetrics();
TopicPartition tp = new TopicPartition(topic, partition); updateMetrics(record, tp); Object value = record.value(); IncomingMessageEnvelope imEnvelope = new IncomingMessageEnvelope(ssp, String.valueOf(record.offset()), key, value, getRecordSize(record), record.timestamp(), Instant.now().toEpochMilli()); messages.add(imEnvelope);
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
/** * Create a KafkaSystemConsumer for the provided {@code systemName} * @param kafkaConsumer kafka Consumer object to be used by this system consumer * @param systemName system name for which we create the consumer * @param config application config * @param clientId clientId from the kafka consumer to be used in the KafkaConsumerProxy * @param metrics metrics for this KafkaSystemConsumer * @param clock system clock */ public KafkaSystemConsumer(Consumer<K, V> kafkaConsumer, String systemName, Config config, String clientId, KafkaSystemConsumerMetrics metrics, Clock clock) { super(metrics.registry(), clock, metrics.getClass().getName()); this.kafkaConsumer = kafkaConsumer; this.clientId = clientId; this.systemName = systemName; this.config = config; this.metrics = metrics; fetchThresholdBytesEnabled = new KafkaConfig(config).isConsumerFetchThresholdBytesEnabled(systemName); // create a sink for passing the messages between the proxy and the consumer messageSink = new KafkaConsumerMessageSink(); // Create the proxy to do the actual message reading. String metricName = String.format("%s-%s", systemName, clientId); proxy = new KafkaConsumerProxy(kafkaConsumer, systemName, clientId, messageSink, metrics, metricName); LOG.info("{}: Created KafkaConsumerProxy {} ", this, proxy); }
response = pollConsumer(sspsToFetch, 500L); List<IncomingMessageEnvelope> envelopes = e.getValue(); if (envelopes != null) { moveMessagesToTheirQueue(e.getKey(), envelopes); populateCurrentLags(sspsToFetch); // find current lags for for each SSP } else { // nothing to read refreshLagMetrics();
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
TopicPartition tp = new TopicPartition(topic, partition); updateMetrics(record, tp); Object value = record.value(); IncomingMessageEnvelope imEnvelope = new IncomingMessageEnvelope(ssp, String.valueOf(record.offset()), key, value, getRecordSize(record)); messages.add(imEnvelope);
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }
/** * Create a KafkaSystemConsumer for the provided {@code systemName} * @param kafkaConsumer kafka Consumer object to be used by this system consumer * @param systemName system name for which we create the consumer * @param config application config * @param clientId clientId from the kafka consumer to be used in the KafkaConsumerProxy * @param metrics metrics for this KafkaSystemConsumer * @param clock system clock */ public KafkaSystemConsumer(Consumer<K, V> kafkaConsumer, String systemName, Config config, String clientId, KafkaSystemConsumerMetrics metrics, Clock clock) { super(metrics.registry(), clock, metrics.getClass().getName()); this.kafkaConsumer = kafkaConsumer; this.clientId = clientId; this.systemName = systemName; this.config = config; this.metrics = metrics; fetchThresholdBytesEnabled = new KafkaConfig(config).isConsumerFetchThresholdBytesEnabled(systemName); // create a sink for passing the messages between the proxy and the consumer messageSink = new KafkaConsumerMessageSink(); // Create the proxy to do the actual message reading. String metricName = String.format("%s-%s", systemName, clientId); proxy = new KafkaConsumerProxy(kafkaConsumer, systemName, clientId, messageSink, metrics, metricName); LOG.info("{}: Created KafkaConsumerProxy {} ", this, proxy); }
response = pollConsumer(sspsToFetch, 500L); List<IncomingMessageEnvelope> envelopes = e.getValue(); if (envelopes != null) { moveMessagesToTheirQueue(e.getKey(), envelopes); populateCurrentLags(sspsToFetch); // find current lags for for each SSP } else { // nothing to read refreshLagMetrics();
proxy.addTopicPartition(topicPartitionsToSSP.get(tp), startingOffset); }); if (proxy != null && !proxy.isRunning()) { LOG.info("{}: Starting proxy {}", this, proxy); proxy.start();
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
private Runnable createProxyThreadRunnable() { Runnable runnable = () -> { isRunning = true; try { consumerPollThreadStartLatch.countDown(); LOG.info("Starting consumer poll thread {} for system {}", consumerPollThread.getName(), systemName); initializeLags(); while (isRunning) { fetchMessages(); } } catch (Throwable throwable) { LOG.error(String.format("Error in KafkaConsumerProxy poll thread for system: %s.", systemName), throwable); // KafkaSystemConsumer uses the failureCause to propagate the throwable to the container failureCause = throwable; isRunning = false; } if (!isRunning) { LOG.info("KafkaConsumerProxy for system {} has stopped.", systemName); } }; return runnable; }
TopicPartition tp = new TopicPartition(topic, partition); updateMetrics(record, tp); Object value = record.value(); IncomingMessageEnvelope imEnvelope = new IncomingMessageEnvelope(ssp, String.valueOf(record.offset()), key, value, getRecordSize(record)); messages.add(imEnvelope);
KafkaConsumerProxy(Consumer<K, V> kafkaConsumer, String systemName, String clientId, KafkaSystemConsumer.KafkaConsumerMessageSink messageSink, KafkaSystemConsumerMetrics samzaConsumerMetrics, String metricName) { this.kafkaConsumer = kafkaConsumer; this.systemName = systemName; this.sink = messageSink; this.kafkaConsumerMetrics = samzaConsumerMetrics; this.metricName = metricName; this.clientId = clientId; this.kafkaConsumerMetrics.registerClientProxy(metricName); consumerPollThread = new Thread(createProxyThreadRunnable()); consumerPollThread.setDaemon(true); consumerPollThread.setName( "Samza KafkaConsumerProxy Poll " + consumerPollThread.getName() + " - " + systemName); LOG.info("Creating KafkaConsumerProxy with systeName={}, clientId={}, metricsName={}", systemName, clientId, metricName); }
private void updateMetrics(ConsumerRecord<K, V> r, TopicPartition tp) { TopicAndPartition tap = KafkaSystemConsumer.toTopicAndPartition(tp); SystemStreamPartition ssp = new SystemStreamPartition(systemName, tp.topic(), new Partition(tp.partition())); Long lag = latestLags.get(ssp); if (lag == null) { throw new SamzaException("Unknown/unregistered ssp in latestLags. ssp=" + ssp + "; system=" + systemName); } long currentSSPLag = lag.longValue(); // lag between the current offset and the highwatermark if (currentSSPLag < 0) { return; } long recordOffset = r.offset(); long highWatermark = recordOffset + currentSSPLag; // derived value for the highwatermark int size = getRecordSize(r); kafkaConsumerMetrics.incReads(tap); kafkaConsumerMetrics.incBytesReads(tap, size); kafkaConsumerMetrics.setOffsets(tap, recordOffset); kafkaConsumerMetrics.incClientBytesReads(metricName, size); kafkaConsumerMetrics.setHighWatermarkValue(tap, highWatermark); }