public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
_numAutoExpandedTopics.inc(); _numAutoExpandedTopicPartitions.inc(mismatchedPartitions); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); } else { numMismatchedTopicPartitions += mismatchedPartitions;
@Override @Put("json") public Representation put(Representation entity) { try { String jsonRequest = entity.getText(); TopicPartition topicPartitionInfo = TopicPartition.init(jsonRequest); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); } if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { _helixMirrorMakerManager.expandTopicInMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully expand topic: %s", topicPartitionInfo)); } else { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to expand topic, topic: %s is not existed!", topicPartitionInfo.getTopic())); } } catch (IOException e) { LOGGER.error("Got error during processing Put request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation( String.format("Failed to expand topic, with exception: %s", e)); } }
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager, 20,
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager, 20,
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(3000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager,
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(2000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager,
public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
_numAutoExpandedTopics.inc(); _numAutoExpandedTopicPartitions.inc(mismatchedPartitions); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); } else { numMismatchedTopicPartitions += mismatchedPartitions;
@Override @Put("json") public Representation put(Representation entity) { try { String jsonRequest = entity.getText(); TopicPartition topicPartitionInfo = TopicPartition.init(jsonRequest); if (_autoTopicWhitelistingManager != null) { _autoTopicWhitelistingManager.removeFromBlacklist(topicPartitionInfo.getTopic()); } if (_helixMirrorMakerManager.isTopicExisted(topicPartitionInfo.getTopic())) { _helixMirrorMakerManager.expandTopicInMirrorMaker(topicPartitionInfo); return new StringRepresentation( String.format("Successfully expand topic: %s", topicPartitionInfo)); } else { getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND); return new StringRepresentation(String.format( "Failed to expand topic, topic: %s is not existed!", topicPartitionInfo.getTopic())); } } catch (Exception e) { LOGGER.error("Got error during processing Put request", e); getResponse().setStatus(Status.SERVER_ERROR_INTERNAL); return new StringRepresentation( String.format("Failed to expand topic, with exception: %s", e)); } }
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager, 20,
String topic = "testTopic" + i; LOGGER.info("Expanding topic: {} , waiting for rebalancing", topic); helixMirrorMakerManager.expandTopicInMirrorMaker(topic, 16); Thread.sleep(5000); assertInstanceOwnedTopicPartitionsBalanced(helixMirrorMakerManager,