public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
"Failed to add new topic: %s, it is already existed!", topicPartitionInfo.getTopic())); } else { _helixMirrorMakerManager.addTopicToMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully add new topic: %s", topicPartitionInfo));
@Test public void testValidation() { helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 1); Assert.assertEquals(validationResultJson.getIntValue("numTopicPartitions"), 1); Assert.assertEquals(validationResultJson.getIntValue("numTopics"), 1); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic1", 2); try { Thread.sleep(3000);
.addFakeDataInstancesToAutoJoinHelixCluster(helixClusterName, ZkStarter.DEFAULT_ZK_STR, 1, 0); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(5000);
.addFakeDataInstancesToAutoJoinHelixCluster(helixClusterName, ZkStarter.DEFAULT_ZK_STR, 1, 0); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(5000);
Assert.assertEquals(validationResult, "{\"mismatchedTopicPartitions\":{},\"numMismatchedTopicPartitions\":0,\"numMismatchedTopics\":0,\"numMissingTopics\":0}"); helixMirrorMakerManager.addTopicToMirrorMaker(topicName, 1); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 1); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 2); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic" + i, 1);
String topic = "testTopic" + i; LOGGER.info("Trying to add topic {}", topic); helixMirrorMakerManager.addTopicToMirrorMaker(topic, 8); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager, 10, 8 * (i + 1));
helixMirrorMakerManager.start(); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(1000);
helixMirrorMakerManager.start(); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(1000);
String topic = "testTopic" + i; LOGGER.info("Trying to add topic {}", topic); helixMirrorMakerManager.addTopicToMirrorMaker(topic, 8); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager, 10, 8 * (i + 1));
String topic = "testTopic" + i; LOGGER.info("Trying to add topic {}", topic); helixMirrorMakerManager.addTopicToMirrorMaker(topic, 8); Thread.sleep(500);
String topic = "testTopic" + i; LOGGER.info("Trying to add topic {}", topic); helixMirrorMakerManager.addTopicToMirrorMaker(topic, 8); Thread.sleep(500);
public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private boolean handleTopicAssignmentOnline(String topic, String srcCluster, String dstCluster) { HelixMirrorMakerManager helixManager = _currentControllerInstance.getHelixResourceManager(); if (helixManager.isTopicExisted(topic)) { LOGGER.warn("Topic {} already exists from cluster {} to {}", topic, srcCluster, dstCluster); return false; } TopicPartition topicPartitionInfo = null; KafkaBrokerTopicObserver topicObserver = _currentControllerInstance.getSourceKafkaTopicObserver(); if (topicObserver == null) { // no source partition information, use partitions=1 and depend on auto-expanding later topicPartitionInfo = new TopicPartition(topic, 1); } else { topicPartitionInfo = topicObserver.getTopicPartitionWithRefresh(topic); if (topicPartitionInfo == null) { String msg = String.format( "Failed to whitelist topic %s on controller because topic does not exists in src cluster %s", topic, srcCluster); LOGGER.error(msg); throw new IllegalArgumentException(msg); } } helixManager.addTopicToMirrorMaker(topicPartitionInfo); LOGGER.info("Whitelisted topic {} from cluster {} to {}", topic, srcCluster, dstCluster); return true; }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
@Test public void testValidation() { helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 1); Assert.assertEquals(validationResultJson.getIntValue("numTopicPartitions"), 1); Assert.assertEquals(validationResultJson.getIntValue("numTopics"), 1); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic1", 2); try { Thread.sleep(3000);
"Failed to add new topic: %s, it is already existed!", topicPartitionInfo.getTopic())); } else { _helixMirrorMakerManager.addTopicToMirrorMaker(topicPartitionInfo); LOGGER.info("successuflly whitelist the topic {}", topicName); return new StringRepresentation(
.addFakeDataInstancesToAutoJoinHelixCluster(helixClusterName, ZkStarter.DEFAULT_ZK_STR, 1, 0); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(5000);
helixMirrorMakerManager.start(); LOGGER.info("Trying to add topic testTopic0"); helixMirrorMakerManager.addTopicToMirrorMaker("testTopic0", 8); Thread.sleep(1000);