private AutoTopicWhitelistingManager getAutoTopicWhitelistingManager() { if (_config.getEnableAutoWhitelist()) { if (!_kafkaBrokerTopicObserverMap.containsKey(SRC_KAFKA_CLUSTER)) { _kafkaBrokerTopicObserverMap.put(SRC_KAFKA_CLUSTER, new KafkaBrokerTopicObserver(SRC_KAFKA_CLUSTER, _config.getSrcKafkaZkPath())); } if (!_kafkaBrokerTopicObserverMap.containsKey(DEST_KAFKA_CLUSTER)) { _kafkaBrokerTopicObserverMap.put(DEST_KAFKA_CLUSTER, new KafkaBrokerTopicObserver(DEST_KAFKA_CLUSTER, _config.getDestKafkaZkPath())); } String patternToExcludeTopics = _config.getPatternToExcludeTopics(); if (patternToExcludeTopics != null && patternToExcludeTopics.trim().length() > 0) { patternToExcludeTopics = patternToExcludeTopics.trim(); } else { patternToExcludeTopics = ""; } LOGGER.info("Pattern to exclude topics is " + patternToExcludeTopics); return new AutoTopicWhitelistingManager(_kafkaBrokerTopicObserverMap.get(SRC_KAFKA_CLUSTER), _kafkaBrokerTopicObserverMap.get(DEST_KAFKA_CLUSTER), _helixMirrorMakerManager, patternToExcludeTopics, _config.getRefreshTimeInSeconds(), _config.getInitWaitTimeInSeconds()); } else { LOGGER.info("Not init AutoTopicWhitelistingManager!"); return null; } }
@Override @Delete public Representation delete() { final String topicName = (String) getRequest().getAttributes().get("topicName"); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.addIntoBlacklist(topicName); } if (!_helixMirrorMakerManager.isTopicExisted(topicName)) { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation( String.format("Failed to delete not existed topic: %s", topicName)); } try { _helixMirrorMakerManager.deleteTopicInMirrorMaker(topicName); return new StringRepresentation( String.format("Successfully finished delete topic: %s", topicName)); } catch (Exception e) { getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); LOGGER.error("Failed to delete topic: {}, with exception: {}", topicName, e); return new StringRepresentation( String.format("Failed to delete topic: %s, with exception: %s", topicName, e)); } }
private Set<String> getCandidateTopicsToWhitelist() { Set<String> candidateTopics = new HashSet<String>(_srcKafkaTopicObserver.getAllTopics()); candidateTopics.retainAll(_destKafkaTopicObserver.getAllTopics()); candidateTopics.removeAll(_helixMirrorMakerManager.getTopicLists()); candidateTopics.addAll(getPartitionMismatchedTopics()); loadBlacklistedTopics(); LOGGER.info("BlacklistedTopics={} and ExcludingPattern={}", _blacklistedTopics, _patternToExcludeTopics); Iterator<String> itr = candidateTopics.iterator(); while (itr.hasNext()) { String topic = itr.next(); if (_blacklistedTopics.contains(topic)) { LOGGER.info("Exclude topic={} by blacklist", topic); itr.remove(); } else if (topic.matches(_patternToExcludeTopics)) { LOGGER.info("Exclude topic={} by pattern", topic); itr.remove(); } } return candidateTopics; }
public void start() { registerMetrics(); LOGGER.info("Creating zkpath={} for blacklisted topics", _blacklistedTopicsZPath); maybeCreateZkPath(_blacklistedTopicsZPath); // Report current status every one minutes. LOGGER.info("Trying to schedule auto topic whitelisting job at rate {} {} !", _refreshTimeInSec, _timeUnit.toString()); _executorService.scheduleAtFixedRate(new Runnable() { @Override public void run() { if (_helixMirrorMakerManager.isLeader()) { _numOfAutoWhitelistingRuns.inc(); LOGGER.info("Trying to run topic whitelisting job"); Set<String> candidateTopicsToWhitelist = null; try { candidateTopicsToWhitelist = getCandidateTopicsToWhitelist(); } catch (Exception e) { LOGGER.error("Failed to get candidate topics: ", e); candidateTopicsToWhitelist = new HashSet<String>(); } LOGGER.info("Trying to whitelist topics: {}", Arrays.toString(candidateTopicsToWhitelist.toArray(new String[0]))); _numErrorTopics.dec(_numErrorTopics.getCount()); whitelistCandiateTopics(candidateTopicsToWhitelist); } else { LOGGER.debug("Not leader, skip auto topic whitelisting!"); } } }, Math.min(_initWaitTimeInSec, _refreshTimeInSec), _refreshTimeInSec, _timeUnit); }
autoTopicWhitelistingManager = new AutoTopicWhitelistingManager(kafkaBrokerTopicObserver, kafkaBrokerTopicObserver, helixMirrorMakerManager, "", 1); autoTopicWhitelistingManager.start();
_validationManager.start(); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.start();
@Override @Put("json") public Representation put(Representation entity) { try { String jsonRequest = entity.getText(); TopicPartition topicPartitionInfo = TopicPartition.init(jsonRequest); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); } if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { _helixMirrorMakerManager.expandTopicInMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully expand topic: %s", topicPartitionInfo)); } else { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to expand topic, topic: %s is not existed!", topicPartitionInfo.getTopic())); } } catch (IOException e) { LOGGER.error("Got error during processing Put request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation( String.format("Failed to expand topic, with exception: %s", e)); } }
public void addIntoBlacklist(String topic) { maybeCreateZkPath(_blacklistedTopicsZPath + "/" + topic); LOGGER.info("topic={} is added to blacklist on zk", topic); }
@BeforeTest public void setup() { LOGGER.info("Trying to setup"); ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); try { Thread.sleep(2000); } catch (Exception e) { } kafkaBrokerTopicObserver = new KafkaBrokerTopicObserver("broker0", KafkaStarterUtils.DEFAULT_ZK_STR, 1); ControllerConf controllerConf = new ControllerConf(); controllerConf.setControllerPort("9090"); controllerConf.setHelixClusterName("TestAutoTopicWhitelistingManager"); controllerConf.setDeploymentName("TestAutoTopicWhitelistingDeployment"); controllerConf.setInstanceId("controller-0"); controllerConf.setControllerMode("customized"); controllerConf.setZkStr(ZkStarter.DEFAULT_ZK_STR); controllerConf.setBackUpToGit("false"); controllerConf.setAutoRebalanceDelayInSeconds("1"); helixMirrorMakerManager = new HelixMirrorMakerManager(controllerConf); helixMirrorMakerManager.start(); autoTopicWhitelistingManager = new AutoTopicWhitelistingManager(kafkaBrokerTopicObserver, kafkaBrokerTopicObserver, helixMirrorMakerManager, "", 1); autoTopicWhitelistingManager.start(); }
public void start() { registerMetrics(); LOGGER.info("Creating zkpath={} for blacklisted topics", _blacklistedTopicsZPath); maybeCreateZkPath(_blacklistedTopicsZPath); // Report current status every one minutes. LOGGER.info("Trying to schedule auto topic whitelisting job at rate {} {} !", _refreshTimeInSec, _timeUnit.toString()); _executorService.scheduleAtFixedRate(new Runnable() { @Override public void run() { if (_helixMirrorMakerManager.isLeader()) { _numOfAutoWhitelistingRuns.inc(); LOGGER.info("Trying to run topic whitelisting job"); Set<String> candidateTopicsToWhitelist = null; try { candidateTopicsToWhitelist = getCandidateTopicsToWhitelist(); } catch (Exception e) { LOGGER.error("Failed to get candidate topics: ", e); candidateTopicsToWhitelist = new HashSet<String>(); } LOGGER.info("Trying to whitelist topics: {}", Arrays.toString(candidateTopicsToWhitelist.toArray(new String[0]))); _numErrorTopics.dec(_numErrorTopics.getCount()); whitelistCandiateTopics(candidateTopicsToWhitelist); } else { LOGGER.debug("Not leader, skip auto topic whitelisting!"); } } }, Math.min(_initWaitTimeInSec, _refreshTimeInSec), _refreshTimeInSec, _timeUnit); }
_validationManager.start(); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.start();
_autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic());
public void addIntoBlacklist(String topic) { maybeCreateZkPath(_blacklistedTopicsZPath + "/" + topic); LOGGER.info("topic={} is added to blacklist on zk", topic); }
private Set<String> getCandidateTopicsToWhitelist() { Set<String> candidateTopics = new HashSet<String>(_srcKafkaTopicObserver.getAllTopics()); candidateTopics.retainAll(_destKafkaTopicObserver.getAllTopics()); candidateTopics.removeAll(_helixMirrorMakerManager.getTopicLists()); candidateTopics.addAll(getPartitionMismatchedTopics()); loadBlacklistedTopics(); LOGGER.info("BlacklistedTopics={} and ExcludingPattern={}", _blacklistedTopics, _patternToExcludeTopics); Iterator<String> itr = candidateTopics.iterator(); while (itr.hasNext()) { String topic = itr.next(); if (_blacklistedTopics.contains(topic)) { LOGGER.info("Exclude topic={} by blacklist", topic); itr.remove(); } else if (topic.matches(_patternToExcludeTopics)) { LOGGER.info("Exclude topic={} by pattern", topic); itr.remove(); } } return candidateTopics; }
private AutoTopicWhitelistingManager getAutoTopicWhitelistingManager() { if (_config.getEnableAutoWhitelist()) { LOGGER.info("Try to init AutoTopicWhitelistingManager!"); if (!_kafkaBrokerTopicObserverMap.containsKey(SRC_KAFKA_CLUSTER)) { _kafkaBrokerTopicObserverMap.put(SRC_KAFKA_CLUSTER, new KafkaBrokerTopicObserver(SRC_KAFKA_CLUSTER, _config.getSrcKafkaZkPath(), TimeUnit.MINUTES.toMillis(5))); } if (!_kafkaBrokerTopicObserverMap.containsKey(DEST_KAFKA_CLUSTER)) { _kafkaBrokerTopicObserverMap.put(DEST_KAFKA_CLUSTER, new KafkaBrokerTopicObserver(DEST_KAFKA_CLUSTER, _config.getDestKafkaZkPath(), TimeUnit.MINUTES.toMillis(5))); } String patternToExcludeTopics = _config.getPatternToExcludeTopics(); if (patternToExcludeTopics != null && patternToExcludeTopics.trim().length() > 0) { patternToExcludeTopics = patternToExcludeTopics.trim(); } else { patternToExcludeTopics = ""; } LOGGER.info("Pattern to exclude topics is " + patternToExcludeTopics); return new AutoTopicWhitelistingManager(_kafkaBrokerTopicObserverMap.get(SRC_KAFKA_CLUSTER), _kafkaBrokerTopicObserverMap.get(DEST_KAFKA_CLUSTER), _helixMirrorMakerManager, patternToExcludeTopics, _config.getWhitelistRefreshTimeInSeconds(), _config.getInitWaitTimeInSeconds()); } else { LOGGER.info("Not init AutoTopicWhitelistingManager!"); return null; } }
@Override @Put("json") public Representation put(Representation entity) { try { String jsonRequest = entity.getText(); TopicPartition topicPartitionInfo = TopicPartition.init(jsonRequest); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); } if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { _helixMirrorMakerManager.expandTopicInMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully expand topic: %s", topicPartitionInfo)); } else { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to expand topic, topic: %s is not existed!", topicPartitionInfo.getTopic())); } } catch (Exception e) { LOGGER.error("Got error during processing Put request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation( String.format("Failed to expand topic, with exception: %s", e)); } }
_autoTopicWhitelistingManager.addIntoBlacklist(topicName);
_autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic());