LOGGER.info("starting helix mirror maker manager"); _helixMirrorMakerManager.start(); _validationManager.start(); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.start();
@Override public void run() { if (_helixMirrorMakerManager.isLeader()) { _isLeaderCounter.inc(1 - _isLeaderCounter.getCount()); LOGGER.info("Trying to run the validation job"); validateExternalView(); } else { cleanupMetrics(); LOGGER.debug("Not leader, skip validation!"); } }
updateIdealstateInfo(topicPartitionMapForIdealState, idealStateForTopic); updateMetrics(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions, numServingTopics, numErrorTopics); updatePerWorkerISMetrics(topicPartitionMapForExternalView); updatePerWorkerEVMetrics(topicPartitionMapForExternalView); constructPerWorkerISCounterJson(topicPartitionMapForIdealState); JSONObject perWorkerEVCounterJson = constructPerWorkerEVCounterJson(topicPartitionMapForExternalView); JSONObject validationResultJson = constructValidationResultJson(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions, numServingTopics, numErrorTopics, perWorkerISCounterJson, perWorkerEVCounterJson);
validationManager = new ValidationManager(helixMirrorMakerManager); validationManager.start();
public ControllerStarter(ControllerConf conf) { LOGGER.info("Trying to init ControllerStarter with config: {}", conf); _config = conf; HelixKafkaMirrorMakerMetricsReporter.init(conf); _component = new Component(); _controllerRestApp = new ControllerRestApplication(null); _helixMirrorMakerManager = new HelixMirrorMakerManager(_config); _validationManager = new ValidationManager(_helixMirrorMakerManager); _srcKafkaValidationManager = getSourceKafkaClusterValidationManager(); _autoTopicWhitelistingManager = getAutoTopicWhitelistingManager(); if (_config.getBackUpToGit()) { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new GitBackUpHandler(conf.getRemoteBackupRepo(), conf.getLocalGitRepoPath()), _config); } else { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new FileBackUpHandler(conf.getLocalBackupFilePath()), _config); } }
private synchronized void updatePerWorkerISMetrics( Map<String, Integer> topicPartitionMapForIdealState) { for (String worker : topicPartitionMapForIdealState.keySet()) { if (!_idealStatePerWorkerTopicPartitionCounter.containsKey(worker)) { Counter workCounter = new Counter(); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().register( getIdealStatePerWorkMetricName(worker), workCounter); } catch (Exception e) { LOGGER.error("Error registering metrics!", e); } _idealStatePerWorkerTopicPartitionCounter.put(worker, workCounter); } Counter counter = _idealStatePerWorkerTopicPartitionCounter.get(worker); counter.inc(topicPartitionMapForIdealState.get(worker) - counter.getCount()); } for (String worker : _idealStatePerWorkerTopicPartitionCounter.keySet()) { if (!topicPartitionMapForIdealState.containsKey(worker)) { Counter counter = _idealStatePerWorkerTopicPartitionCounter.get(worker); counter.dec(counter.getCount()); } } }
private synchronized void updatePerWorkerEVMetrics( Map<String, Integer> topicPartitionMapForExternalView) { for (String worker : topicPartitionMapForExternalView.keySet()) { if (!_externalViewPerWorkerTopicPartitionCounter.containsKey(worker)) { Counter workCounter = new Counter(); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().register( getExternalViewPerWorkMetricName(worker), workCounter); } catch (Exception e) { LOGGER.error("Error registering metrics!", e); } _externalViewPerWorkerTopicPartitionCounter.put(worker, workCounter); } Counter counter = _externalViewPerWorkerTopicPartitionCounter.get(worker); counter.inc(topicPartitionMapForExternalView.get(worker) - counter.getCount()); } for (String worker : _externalViewPerWorkerTopicPartitionCounter.keySet()) { if (!topicPartitionMapForExternalView.containsKey(worker)) { Counter counter = _externalViewPerWorkerTopicPartitionCounter.get(worker); counter.dec(counter.getCount()); } } }
public void start() { registerMetrics();
validationManager = new ValidationManager(helixMirrorMakerManager); validationManager.start();
public ControllerInstance(ManagerControllerHelix managerControllerHelix, ControllerConf conf) { LOGGER.info("Trying to init ControllerStarter with config: {}", conf); _managerControllerHelix = managerControllerHelix; _config = conf; HelixKafkaMirrorMakerMetricsReporter.init(conf); _component = new Component(); _controllerRestApp = new ControllerRestApplication(null); _helixMirrorMakerManager = new HelixMirrorMakerManager(_config); _validationManager = new ValidationManager(_helixMirrorMakerManager); _srcKafkaValidationManager = getSourceKafkaClusterValidationManager(); _autoTopicWhitelistingManager = getAutoTopicWhitelistingManager(); if (_config.getBackUpToGit()) { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new GitBackUpHandler(conf.getRemoteBackupRepo(), conf.getLocalGitRepoPath()), _config); } else { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new FileBackUpHandler(conf.getLocalBackupFilePath()), _config); } }
private synchronized void updatePerWorkerISMetrics( Map<String, Integer> topicPartitionMapForIdealState) { for (String worker : topicPartitionMapForIdealState.keySet()) { if (!_idealStatePerWorkerTopicPartitionCounter.containsKey(worker)) { Counter workCounter = new Counter(); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().register( getIdealStatePerWorkMetricName(worker), workCounter); } catch (Exception e) { LOGGER.error("Error registering metrics!", e); } _idealStatePerWorkerTopicPartitionCounter.put(worker, workCounter); } Counter counter = _idealStatePerWorkerTopicPartitionCounter.get(worker); counter.inc(topicPartitionMapForIdealState.get(worker) - counter.getCount()); } for (String worker : _idealStatePerWorkerTopicPartitionCounter.keySet()) { if (!topicPartitionMapForIdealState.containsKey(worker)) { //Counter counter = _idealStatePerWorkerTopicPartitionCounter.get(worker); //counter.dec(counter.getCount()); _idealStatePerWorkerTopicPartitionCounter.remove(worker); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().remove(getIdealStatePerWorkMetricName(worker)); } catch (Exception e) { LOGGER.warn("Got exception when removing metrics for {}", getIdealStatePerWorkMetricName(worker), e); } } } }
private synchronized void updatePerWorkerEVMetrics( Map<String, Integer> topicPartitionMapForExternalView) { for (String worker : topicPartitionMapForExternalView.keySet()) { if (!_externalViewPerWorkerTopicPartitionCounter.containsKey(worker)) { Counter workCounter = new Counter(); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().register( getExternalViewPerWorkMetricName(worker), workCounter); } catch (Exception e) { LOGGER.error("Error registering metrics!", e); } _externalViewPerWorkerTopicPartitionCounter.put(worker, workCounter); } Counter counter = _externalViewPerWorkerTopicPartitionCounter.get(worker); counter.inc(topicPartitionMapForExternalView.get(worker) - counter.getCount()); } for (String worker : _externalViewPerWorkerTopicPartitionCounter.keySet()) { if (!topicPartitionMapForExternalView.containsKey(worker)) { //Counter counter = _externalViewPerWorkerTopicPartitionCounter.get(worker); //counter.dec(counter.getCount()); _externalViewPerWorkerTopicPartitionCounter.remove(getExternalViewPerWorkMetricName(worker)); try { HelixKafkaMirrorMakerMetricsReporter.get().getRegistry().remove(getExternalViewPerWorkMetricName(worker)); } catch (Exception e) { LOGGER.warn("Got exception when removing metrics for {}", getExternalViewPerWorkMetricName(worker), e); } } } }
public void start() { registerMetrics();
updateIdealstateInfo(topicPartitionMapForIdealState, idealStateForTopic); updateMetrics(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions, numServingTopics, numErrorTopics); updatePerWorkerISMetrics(topicPartitionMapForIdealState); updatePerWorkerEVMetrics(topicPartitionMapForExternalView); constructPerWorkerISCounterJson(topicPartitionMapForIdealState); JSONObject perWorkerEVCounterJson = constructPerWorkerEVCounterJson(topicPartitionMapForExternalView); JSONObject validationResultJson = constructValidationResultJson(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions, numServingTopics, numErrorTopics, perWorkerISCounterJson, perWorkerEVCounterJson);
@Override @Get public Representation get() { final String option = (String) getRequest().getAttributes().get("option"); if ("srcKafka".equals(option)) { if (_srcKafkaValidationManager == null) { LOGGER.warn("SourceKafkaClusterValidationManager is null!"); return new StringRepresentation("SrcKafkaValidationManager is not been initialized!"); } LOGGER.info("Trying to call validation on source kafka cluster!"); return new StringRepresentation(_srcKafkaValidationManager.validateSourceKafkaCluster()); } else { LOGGER.info("Trying to call validation on current cluster!"); return new StringRepresentation(_validationManager.validateExternalView()); } }
LOGGER.info("starting helix mirror maker manager"); _helixMirrorMakerManager.start(); _validationManager.start(); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.start();
JSONObject.parseObject(validationManager.validateExternalView()); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopicPartitions"), 0); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopics"), 0); } catch (Exception e) { validationResultJson = JSONObject.parseObject(validationManager.validateExternalView()); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopicPartitions"), 0); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopics"), 0);
@Override public void run() { if (_helixMirrorMakerManager.isLeader()) { _isLeaderCounter.inc(1 - _isLeaderCounter.getCount()); LOGGER.info("Trying to run the validation job"); validateExternalView(); } else { cleanupMetrics(); LOGGER.debug("Not leader, skip validation!"); } }
@Override @Get public Representation get() { final String option = (String) getRequest().getAttributes().get("option"); if ("srcKafka".equals(option)) { if (_srcKafkaValidationManager == null) { LOGGER.warn("SourceKafkaClusterValidationManager is null!"); return new StringRepresentation("SrcKafkaValidationManager is not been initialized!"); } LOGGER.info("Trying to call validation on source kafka cluster!"); return new StringRepresentation(_srcKafkaValidationManager.validateSourceKafkaCluster()); } else { LOGGER.info("Trying to call validation on current cluster!"); return new StringRepresentation(_validationManager.validateExternalView()); } }
JSONObject.parseObject(validationManager.validateExternalView()); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopicPartitions"), 0); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopics"), 0); } catch (Exception e) { validationResultJson = JSONObject.parseObject(validationManager.validateExternalView()); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopicPartitions"), 0); Assert.assertEquals(validationResultJson.getIntValue("numErrorTopics"), 0);