public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
public OnlineOfflineStateModel() { super(generateConfigForOnlineOffline()); }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
public TopicPartition getTopicPartition(String topic) { TopicPartition topicPartition = _topicPartitionInfoMap.get(topic); if (topicPartition != null) { return new TopicPartition(topic, topicPartition.getPartition()); } else { return null; } }
private static Set<InstanceTopicPartitionHolder> balanceAssignment( TreeSet<InstanceTopicPartitionHolder> orderedSet) { while (!isAssignmentBalanced(orderedSet)) { InstanceTopicPartitionHolder lowestInstance = orderedSet.pollFirst(); InstanceTopicPartitionHolder highestInstance = orderedSet.pollLast(); TopicPartition tpi = highestInstance.getServingTopicPartitionSet().iterator().next(); highestInstance.removeTopicPartition(tpi); lowestInstance.addTopicPartition(tpi); orderedSet.add(lowestInstance); orderedSet.add(highestInstance); } return orderedSet; }
public synchronized void addTopicToMirrorMaker(String topicName, int numTopicPartitions) { setEmptyResourceConfig(topicName); updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.addResource(_helixClusterName, topicName, IdealStateBuilder.buildCustomIdealStateFor(topicName, numTopicPartitions, _currentServingInstance)); } }
public Request getTopicCreationRequestUrl(String topic, int numPartitions) { Request request = new Request(Method.POST, _baseUrl + "/topics/"); TopicPartition topicPartitionInfo = new TopicPartition(topic, numPartitions); request.setEntity(topicPartitionInfo.toJSON().toJSONString(), MediaType.APPLICATION_JSON); return request; }
@Override public void run() { try { rebalanceCurrentCluster(_helixMirrorMakerManager.getCurrentLiveInstances()); } catch (Exception e) { LOGGER.error("Got exception during rebalance the whole cluster! ", e); } } }, _delayedAutoReblanceTimeInSeconds, TimeUnit.SECONDS);
@Override public int compare(InstanceTopicPartitionHolder o1, InstanceTopicPartitionHolder o2) { int size1 = (o1 == null) ? -1 : o1.getNumServingTopicPartitions(); int size2 = (o2 == null) ? -1 : o2.getNumServingTopicPartitions(); if (size1 != size2) { return size1 - size2; } else { return o1.getInstanceName().compareTo(o2.getInstanceName()); } } };
@SuppressWarnings("unused") private static boolean isAnyWorkerDown(List<LiveInstance> liveInstances, Map<String, Set<TopicPartition>> instanceToTopicPartitionMap) { Set<String> removedInstances = getRemovedInstanceSet(getLiveInstanceName(liveInstances), instanceToTopicPartitionMap.keySet()); return !removedInstances.isEmpty(); }
public synchronized void expandTopicInMirrorMaker(String topicName, int newNumTopicPartitions) { updateCurrentServingInstance(); synchronized (_currentServingInstance) { _helixAdmin.setResourceIdealState(_helixClusterName, topicName, IdealStateBuilder.expandCustomRebalanceModeIdealStateFor( _helixAdmin.getResourceIdealState(_helixClusterName, topicName), topicName, newNumTopicPartitions, _currentServingInstance)); } }
public AutoRebalanceLiveInstanceChangeListener(HelixMirrorMakerManager helixMirrorMakerManager, HelixManager helixManager, int delayedAutoReblanceTimeInSeconds) { _helixMirrorMakerManager = helixMirrorMakerManager; _helixManager = helixManager; _delayedAutoReblanceTimeInSeconds = delayedAutoReblanceTimeInSeconds; LOGGER.info("Delayed Auto Reblance Time In Seconds: {}", _delayedAutoReblanceTimeInSeconds); registerMetrics(); }
private static boolean isAssignmentBalanced(TreeSet<InstanceTopicPartitionHolder> set) { return (set.last().getNumServingTopicPartitions() - set.first().getNumServingTopicPartitions() <= 1); }
public void addIntoBlacklist(String topic) { maybeCreateZkPath(_blacklistedTopicsZPath + "/" + topic); LOGGER.info("topic={} is added to blacklist on zk", topic); }
@Override public void run() { try { dumpState(); } catch (Exception e) { LOGGER.error(String.format("Failed to take backup, with exception: %s", e)); return; } LOGGER.info("Backup taken successfully!"); } }, 20, _timeValue, _timeUnit);