private Set<String> getCandidateTopicsToWhitelist() { Set<String> candidateTopics = new HashSet<String>(_srcKafkaTopicObserver.getAllTopics()); candidateTopics.retainAll(_destKafkaTopicObserver.getAllTopics()); candidateTopics.removeAll(_helixMirrorMakerManager.getTopicLists()); candidateTopics.addAll(getPartitionMismatchedTopics()); loadBlacklistedTopics(); LOGGER.info("BlacklistedTopics={} and ExcludingPattern={}", _blacklistedTopics, _patternToExcludeTopics); Iterator<String> itr = candidateTopics.iterator(); while (itr.hasNext()) { String topic = itr.next(); if (_blacklistedTopics.contains(topic)) { LOGGER.info("Exclude topic={} by blacklist", topic); itr.remove(); } else if (topic.matches(_patternToExcludeTopics)) { LOGGER.info("Exclude topic={} by pattern", topic); itr.remove(); } } return candidateTopics; }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
Map<String, Integer> misMatchedPartitionNumberTopics = new HashMap<String, Integer>(); int numMismatchedTopicPartitions = 0; for (String topic : _helixMirrorMakerManager.getTopicLists()) { TopicPartition tp = _sourceKafkaTopicObserver.getTopicPartition(topic); if (tp == null) {
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
int numServingTopics = 0; int numErrorTopics = 0; for (String topicName : _helixMirrorMakerManager.getTopicLists()) { numServingTopics++; IdealState idealStateForTopic =
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
StringBuilder idealState = new StringBuilder(); StringBuilder partitionAssignment = new StringBuilder(); List<String> topicLists = _helixMirrorMakerManager.getTopicLists(); if (topicLists == null || topicLists.isEmpty()) { LOGGER.info("No topics available to take backup");
final String topicName = (String) getRequest().getAttributes().get("topicName"); if (topicName == null) { List<String> topicLists = _helixMirrorMakerManager.getTopicLists(); if (topicLists == null || topicLists.isEmpty()) { return new StringRepresentation("No topic is added in MirrorMaker Controller!");
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
"{\"mismatchedTopicPartitions\":{},\"numMismatchedTopicPartitions\":0,\"numMismatchedTopics\":0,\"numMissingTopics\":0}"); for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; Assert.assertEquals(validationResultJson.get("numMissingTopics"), 10); for (int i = 10; i < 20; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), 20); String topicName = "testTopic" + i; Assert.assertEquals(validationResultJson.get("numMissingTopics"), 10); for (int i = 20; i < 30; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), 30); String topicName = "testTopic" + i;
private Set<String> getCandidateTopicsToWhitelist() { Set<String> candidateTopics = new HashSet<String>(_srcKafkaTopicObserver.getAllTopics()); candidateTopics.retainAll(_destKafkaTopicObserver.getAllTopics()); candidateTopics.removeAll(_helixMirrorMakerManager.getTopicLists()); candidateTopics.addAll(getPartitionMismatchedTopics()); loadBlacklistedTopics(); LOGGER.info("BlacklistedTopics={} and ExcludingPattern={}", _blacklistedTopics, _patternToExcludeTopics); Iterator<String> itr = candidateTopics.iterator(); while (itr.hasNext()) { String topic = itr.next(); if (_blacklistedTopics.contains(topic)) { LOGGER.info("Exclude topic={} by blacklist", topic); itr.remove(); } else if (topic.matches(_patternToExcludeTopics)) { LOGGER.info("Exclude topic={} by pattern", topic); itr.remove(); } } return candidateTopics; }
public void refreshWorkloads() throws IOException { long current = System.currentTimeMillis(); if (_lastRefreshTimeMillis + _minRefreshIntervalMillis > current) { LOGGER.info("Too soon to refresh workload, skip"); return; } LOGGER.info("Refreshing workload for source " + _srcKafkaCluster); _lastRefreshTimeMillis = current; List<String> topics = _helixMirrorMakerManager.getTopicLists(); Map<String, Integer> topicsPartitions = new HashMap<>(); for (String topic : topics) { IdealState idealState = _helixMirrorMakerManager.getIdealStateForTopic(topic); if (idealState != null) { int partitions = idealState.getNumPartitions(); if (partitions > 0) { topicsPartitions.put(topic, partitions); } } } retrieveWorkload(current, DEFAULT_WORKLOAD_WINDOW_MILLIS, topicsPartitions); }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
private void assertEmptyCluster(HelixMirrorMakerManager helixMirrorMakerManager) { for (String topicName : helixMirrorMakerManager.getTopicLists()) { Assert.assertNull( helixMirrorMakerManager.getExternalViewForTopic(topicName)); Assert.assertEquals( helixMirrorMakerManager.getIdealStateForTopic(topicName).getPartitionSet().size(), 0); } }
private void assertInstanceOwnedTopicPartitionsBalanced( HelixMirrorMakerManager helixMirrorMakerManager, int numInstances, int numTotalPartitions) { Map<String, Integer> serverToPartitionMapping = new HashMap<String, Integer>(); for (String topicName : helixMirrorMakerManager.getTopicLists()) { ExternalView externalViewForTopic = helixMirrorMakerManager.getExternalViewForTopic(topicName); LOGGER.info("ExternalView: " + externalViewForTopic.toString()); for (String partition : externalViewForTopic.getPartitionSet()) { String instanceName = externalViewForTopic.getStateMap(partition).keySet().iterator().next(); if (!serverToPartitionMapping.containsKey(instanceName)) { serverToPartitionMapping.put(instanceName, 0); } serverToPartitionMapping.put(instanceName, serverToPartitionMapping.get(instanceName) + 1); } } int expectedLowerBound = (int) Math.floor((double) numTotalPartitions / (double) numInstances); int expectedUpperBound = (int) Math.ceil((double) numTotalPartitions / (double) numInstances); for (String instanceName : serverToPartitionMapping.keySet()) { // May not be perfect balancing. LOGGER.info("Current {} serving {} partitions, expected [{}, {}]", instanceName, serverToPartitionMapping.get(instanceName), expectedLowerBound, expectedUpperBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) >= expectedLowerBound); Assert.assertTrue(serverToPartitionMapping.get(instanceName) <= expectedUpperBound); } }
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }