protected void updateOffset() { logger.debug("OffsetMonitor updates offset with leaders=" + partitionLeader); offsetMonitorFailureCount.set(0); for (Map.Entry<TopicAndPartition, BrokerEndPoint> entry : partitionLeader.entrySet()) { String leaderBroker = getHostPort(entry.getValue()); TopicAndPartition tp = entry.getKey(); if (StringUtils.isEmpty(leaderBroker)) { logger.warn("{} does not have leader partition", tp); } else { try { cronExecutor.submit(updateOffsetTask(leaderBroker, tp)); } catch (RejectedExecutionException re) { offsetMonitorFailureCount.getAndAdd(1); logger.warn(String.format("cronExecutor is full! Drop task for topic: %s, partition: %d", tp.topic(), tp.partition()), re); throw re; } catch (Throwable t) { offsetMonitorFailureCount.getAndAdd(1); logger.error(String.format("cronExecutor got throwable! Drop task for topic: %s, partition: %d", tp.topic(), tp.partition()), t); throw t; } } } }
Map<String, Gauge> gauges = metricRegistry.getGauges(); for (final TopicAndPartition topicPartition : topicPartitionToOffsetMap.keySet()) { String metricName = getOffsetLagName(topicPartition); if (!gauges.containsKey(metricName)) { Gauge<Long> gauge = new Gauge<Long>() { List<TopicAndPartition> noProgressPartitions = getNoProgessTopicPartitions(); numNoProgressTopicPartitions.set(noProgressPartitions.size()); if (!noProgressPartitions.isEmpty()) {
/** * Get stuck topic partitions via offset manager. * * @return the topic partitions that have been stuck for at least _movePartitionAfterStuckMillis. */ private Set<TopicPartition> getStuckTopicPartitions() { Set<TopicPartition> partitions = new HashSet<>(); if (_movePartitionAfterStuckMillis <= 0) { return partitions; } Map<TopicAndPartition, TopicPartitionLag> noProgressMap = _helixMirrorMakerManager.getOffsetMonitor() .getNoProgressTopicToOffsetMap(); long now = System.currentTimeMillis(); for (Map.Entry<TopicAndPartition, TopicPartitionLag> entry : noProgressMap.entrySet()) { TopicPartitionLag lastLag = entry.getValue(); if (now - lastLag.getTimeStamp() > _movePartitionAfterStuckMillis) { partitions.add(new TopicPartition(entry.getKey().topic(), entry.getKey().partition())); } } return partitions; }
registerNoProgressMetric(); registerUpdateOffsetStatusMetric(); } else { logger.info("OffsetMonitor is disabled");
@Override @Get public Representation get() { OffsetMonitor offsetMonitor = _helixMirrorMakerManager.getOffsetMonitor(); final String topicName = (String) getRequest().getAttributes().get("topic"); if (topicName == null) { try { return new StringRepresentation(offsetMonitor.getTopicToOffsetMap().toString()); } catch (Exception e) { LOGGER.error("Got error during processing Get request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation(String .format("Failed to get lag for offset map for all partitions with exception: %s", e)); } } final int partition = Integer.parseInt((String) getRequest().getAttributes().get("partition")); try { return new StringRepresentation(offsetMonitor.getTopicPartitionOffset(new TopicPartition(topicName, partition)).toString()); } catch (Exception e) { LOGGER.error("Got error during processing Get request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation(String .format("Failed to get offset for topic partition: %s:%d, with exception: %s", topicName, partition, e)); } }
SimpleConsumer consumer = getSimpleConsumer(broker); TopicMetadataRequest req = new TopicMetadataRequest(topicList); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
public HelixMirrorMakerManager(ControllerConf controllerConf) { _controllerConf = controllerConf; _helixZkURL = HelixUtils.getAbsoluteZkPathForHelix(_controllerConf.getZkStr()); _helixClusterName = _controllerConf.getHelixClusterName(); _instanceId = controllerConf.getInstanceId(); _workloadInfoRetriever = new WorkloadInfoRetriever(this); _currentServingInstance = new PriorityQueue<>(1, InstanceTopicPartitionHolder.getTotalWorkloadComparator(_workloadInfoRetriever, null)); _offsetMonitor = new OffsetMonitor(this, controllerConf); }
/** * Return the lagging time if the given partition has lag. * * @param tp topic partition * @return the lagging time in seconds if the given partition has lag; otherwise return 0. */ private long getLagTime(TopicPartition tp) { TopicPartitionLag tpl = _helixMirrorMakerManager.getOffsetMonitor().getTopicPartitionOffset(tp); if (tpl == null || tpl.getLatestOffset() <= 0 || tpl.getCommitOffset() <= 0 || System.currentTimeMillis() - tpl.getTimeStamp() > _offsetMaxValidTimeMillis) { return 0; } long lag = tpl.getLatestOffset() - tpl.getCommitOffset(); if (lag <= _minLagOffset) { return 0; } double msgRate = _helixMirrorMakerManager.getWorkloadInfoRetriever().topicWorkload(tp.getTopic()) .getMsgsPerSecondPerPartition(); if (msgRate < 1) { msgRate = 1; } double lagTime = lag / msgRate; if (lagTime > _minLagTimeSec) { return Math.round(lagTime); } return 0; }