controllerConf.setBackUpToGit("false"); controllerConf.setAutoRebalanceDelayInSeconds("0"); HelixMirrorMakerManager helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start(); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(1000); Thread.sleep(5000); assertTopicExternalViewWithGivenPartitions(helixMirrorMakerManager, "testTopic0", 8, 2); helixMirrorMakerManager.stop();
@Override @Delete public Representation delete() { final String topicName = (String) getRequest().getAttributes().get("topicName"); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.addIntoBlacklist(topicName); } if (!_helixMirrorMakerManager.isTopicExisted(topicName)) { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation( String.format("Failed to delete not existed topic: %s", topicName)); } try { _helixMirrorMakerManager.deleteTopicInMirrorMaker(topicName); return new StringRepresentation( String.format("Successfully finished delete topic: %s", topicName)); } catch (Exception e) { getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); LOGGER.error("Failed to delete topic: {}, with exception: {}", topicName, e); return new StringRepresentation( String.format("Failed to delete topic: %s, with exception: %s", topicName, e)); } }
Map<String, Integer> misMatchedPartitionNumberTopics = new HashMap<String, Integer>(); int numMismatchedTopicPartitions = 0; for (String topic : _helixMirrorMakerManager.getTopicLists()) { TopicPartition tp = _sourceKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { } else { int numPartitionsInMirrorMaker = _helixMirrorMakerManager.getIdealStateForTopic(topic).getNumPartitions(); if (numPartitionsInMirrorMaker != tp.getPartition()) { int mismatchedPartitions = Math.abs(numPartitionsInMirrorMaker - tp.getPartition()); _numAutoExpandedTopics.inc(); _numAutoExpandedTopicPartitions.inc(mismatchedPartitions); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); } else { numMismatchedTopicPartitions += mismatchedPartitions; misMatchedPartitionNumberTopics.size(), numMismatchedTopicPartitions, mismatchedTopicPartitionsJson); if (_helixMirrorMakerManager.isLeader()) { updateMetrics(notExistedTopics.size(), misMatchedPartitionNumberTopics.size(), numMismatchedTopicPartitions, misMatchedPartitionNumberTopics);
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
int numServingTopics = 0; int numErrorTopics = 0; for (String topicName : _helixMirrorMakerManager.getTopicLists()) { numServingTopics++; IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName); numTopicPartitions += idealStateForTopic.getNumPartitions(); if (idealStateForTopic.getNumPartitions() != externalViewForTopic.getPartitionSet() if (_helixMirrorMakerManager.isLeader()) { updateMetrics(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions, numServingTopics, numErrorTopics);
final String topicName = (String) getRequest().getAttributes().get("topicName"); if (topicName == null) { List<String> topicLists = _helixMirrorMakerManager.getTopicLists(); if (topicLists == null || topicLists.isEmpty()) { return new StringRepresentation("No topic is added in MirrorMaker Controller!"); if (_helixMirrorMakerManager.isTopicExisted(topicName)) { IdealState idealStateForTopic = _helixMirrorMakerManager.getIdealStateForTopic(topicName); ExternalView externalViewForTopic = _helixMirrorMakerManager.getExternalViewForTopic(topicName); JSONObject responseJson = new JSONObject(); responseJson.put("topic", topicName);
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
controllerConf.setAutoRebalanceDelayInSeconds("1"); helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start();
"{\"mismatchedTopicPartitions\":{},\"numMismatchedTopicPartitions\":0,\"numMismatchedTopics\":0,\"numMissingTopics\":0}"); for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; Assert.assertEquals(validationResult, "{\"mismatchedTopicPartitions\":{},\"numMismatchedTopicPartitions\":0,\"numMismatchedTopics\":0,\"numMissingTopics\":0}"); helixMirrorMakerManager.addTopicToMirrorMaker(topicName, 1); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 1); Assert.assertEquals(validationResultJson.get("numMissingTopics"), 10); for (int i = 10; i < 20; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), 20); String topicName = "testTopic" + i; helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 2); Assert.assertEquals(validationResultJson.get("numMissingTopics"), 10); for (int i = 20; i < 30; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), 30); String topicName = "testTopic" + i; helixMirrorMakerManager.deleteTopicInMirrorMaker("testTopic" + i); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 1);
public void stop() { try { LOGGER.info("stopping broker topic observers"); for (String key : _kafkaBrokerTopicObserverMap.keySet()) { try { KafkaBrokerTopicObserver observer = _kafkaBrokerTopicObserverMap.get(key); observer.stop(); } catch (Exception e) { LOGGER.error("Failed to stop KafkaBrokerTopicObserver: {}!", key); } } LOGGER.info("stopping api component"); _component.stop(); LOGGER.info("stopping resource manager"); _helixMirrorMakerManager.stop(); } catch (final Exception e) { LOGGER.error("Caught exception", e); } }
_autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to add new topic: %s, it is already existed!", topicPartitionInfo.getTopic())); } else { _helixMirrorMakerManager.addTopicToMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully add new topic: %s", topicPartitionInfo));
private Set<String> getCandidateTopicsToWhitelist() { Set<String> candidateTopics = new HashSet<String>(_srcKafkaTopicObserver.getAllTopics()); candidateTopics.retainAll(_destKafkaTopicObserver.getAllTopics()); candidateTopics.removeAll(_helixMirrorMakerManager.getTopicLists()); candidateTopics.addAll(getPartitionMismatchedTopics()); loadBlacklistedTopics(); LOGGER.info("BlacklistedTopics={} and ExcludingPattern={}", _blacklistedTopics, _patternToExcludeTopics); Iterator<String> itr = candidateTopics.iterator(); while (itr.hasNext()) { String topic = itr.next(); if (_blacklistedTopics.contains(topic)) { LOGGER.info("Exclude topic={} by blacklist", topic); itr.remove(); } else if (topic.matches(_patternToExcludeTopics)) { LOGGER.info("Exclude topic={} by pattern", topic); itr.remove(); } } return candidateTopics; }
@Override @Put("json") public Representation put(Representation entity) { try { String jsonRequest = entity.getText(); TopicPartition topicPartitionInfo = TopicPartition.init(jsonRequest); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); } if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { _helixMirrorMakerManager.expandTopicInMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully expand topic: %s", topicPartitionInfo)); } else { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to expand topic, topic: %s is not existed!", topicPartitionInfo.getTopic())); } } catch (IOException e) { LOGGER.error("Got error during processing Put request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation( String.format("Failed to expand topic, with exception: %s", e)); } }
public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private void assertTopicExternalViewWithGivenPartitions( HelixMirrorMakerManager helixMirrorMakerManager, String topicName, int partitions, int numInstances) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); Assert.assertEquals(externalViewForTopic.getPartitionSet().size(), partitions); Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } int expectedLowerBound = (int) Math.floor((double) partitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) partitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound - 1); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound + 1); } } }
@Override @Get public Representation get() { final String opt = (String) getRequest().getAttributes().get("opt"); if ("disable_autobalancing".equalsIgnoreCase(opt)) { _helixMirrorMakerManager.disableAutoBalancing(); LOGGER.info("Disabled autobalancing!"); return new StringRepresentation("Disabled autobalancing!\n"); } else if ("enable_autobalancing".equalsIgnoreCase(opt)) { _helixMirrorMakerManager.enableAutoBalancing(); LOGGER.info("Enabled autobalancing!"); return new StringRepresentation("Enabled autobalancing!\n"); } else if ("autobalancing_status".equalsIgnoreCase(opt)) { if (_helixMirrorMakerManager.isAutoBalancingEnabled()) { return new StringRepresentation("enabled"); } else { return new StringRepresentation("disabled"); } } LOGGER.info("No valid input!"); return new StringRepresentation("No valid input!\n"); }
public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
public ControllerStarter(ControllerConf conf) { LOGGER.info("Trying to init ControllerStarter with config: {}", conf); _config = conf; HelixKafkaMirrorMakerMetricsReporter.init(conf); _component = new Component(); _controllerRestApp = new ControllerRestApplication(null); _helixMirrorMakerManager = new HelixMirrorMakerManager(_config); _validationManager = new ValidationManager(_helixMirrorMakerManager); _srcKafkaValidationManager = getSourceKafkaClusterValidationManager(); _autoTopicWhitelistingManager = getAutoTopicWhitelistingManager(); if (_config.getBackUpToGit()) { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new GitBackUpHandler(conf.getRemoteBackupRepo(), conf.getLocalGitRepoPath()), _config); } else { _clusterInfoBackupManager = new ClusterInfoBackupManager(_helixMirrorMakerManager, new FileBackUpHandler(conf.getLocalBackupFilePath()), _config); } }