public TopicPartition getTopicPartition(String topic) { TopicPartition topicPartition = _topicPartitionInfoMap.get(topic); if (topicPartition != null) { return new TopicPartition(topic, topicPartition.getPartition()); } else { return null; } }
/** * This is used only for POST and PUT call to create the pojo. * * @param jsonRequest * @return */ public static TopicPartition init(String jsonRequest) { JSONObject jsonObject = JSON.parseObject(jsonRequest); if (!jsonObject.containsKey("topic")) { throw new RuntimeException("Cannot initialize TopicPartitionInfo, missing field: topic"); } if (!jsonObject.containsKey("numPartitions")) { throw new RuntimeException( "Cannot initialize TopicPartitionInfo, missing field: numPartitions"); } return new TopicPartition(jsonObject.getString("topic"), jsonObject.getIntValue("numPartitions")); }
scala.collection.Map<Object, Seq<Object>> partitionsMap = partitionAssignmentForTopics.get(topic).get(); TopicPartition tp = new TopicPartition(topic, partitionsMap.size()); _topicPartitionInfoMap.put(topic, tp); } catch (Exception e) {
@Override public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception { if (!tryToRefreshCache()) { synchronized (_lock) { Set<String> newAddedTopics = new HashSet<String>(currentChilds); Set<String> currentServingTopics = getAllTopics(); newAddedTopics.removeAll(currentServingTopics); for (String existedTopic : currentServingTopics) { if (!currentChilds.contains(existedTopic)) { _topicPartitionInfoMap.remove(existedTopic); } } scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>> partitionAssignmentForTopics = _zkUtils.getPartitionAssignmentForTopics( JavaConversions.asScalaBuffer(ImmutableList.copyOf(newAddedTopics))); for (String topic : newAddedTopics) { try { scala.collection.Map<Object, Seq<Object>> partitionsMap = partitionAssignmentForTopics.get(topic).get(); TopicPartition tp = new TopicPartition(topic, partitionsMap.size()); _topicPartitionInfoMap.put(topic, tp); } catch (Exception e) { LOGGER.warn("Failed to get topicPartition info for {} from kafka zk: {}", topic, e); } } _kafkaTopicsCounter.inc(_topicPartitionInfoMap.size() - _kafkaTopicsCounter.getCount()); } } }
/** * From IdealStates. * @param helixManager * @return InstanceToNumTopicPartitionMap */ public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap( HelixManager helixManager) { Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap = new HashMap<String, Set<TopicPartition>>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); for (String partition : is.getPartitionSet()) { TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition)); for (String instance : is.getInstanceSet(partition)) { if (!instanceToNumTopicPartitionMap.containsKey(instance)) { instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>()); } instanceToNumTopicPartitionMap.get(instance).add(tpi); } } } return instanceToNumTopicPartitionMap; }
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState, String topicName, int newNumTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(newNumTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(newNumTopicPartitions); int numOldPartitions = oldIdealState.getNumPartitions(); for (int i = 0; i < numOldPartitions; ++i) { String partitionName = Integer.toString(i); try { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE"); } catch (Exception e) { // No worker added into the cluster. } } for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } return customModeIdealStateBuilder.build(); }
public Request getTopicCreationRequestUrl(String topic, int numPartitions) { Request request = new Request(Method.POST, _baseUrl + "/topics/"); TopicPartition topicPartitionInfo = new TopicPartition(topic, numPartitions); request.setEntity(topicPartitionInfo.toJSON().toJSONString(), MediaType.APPLICATION_JSON); return request; }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public Request getTopicExpansionRequestUrl(String topic, int numPartitions) { Request request = new Request(Method.PUT, _baseUrl + "/topics/"); TopicPartition topicPartitionInfo = new TopicPartition(topic, numPartitions); request.setEntity(topicPartitionInfo.toJSON().toJSONString(), MediaType.APPLICATION_JSON); return request; } }
public TopicPartition getTopicPartition(String topic) { TopicPartition topicPartition = _topicPartitionInfoMap.get(topic); if (topicPartition != null) { return new TopicPartition(topic, topicPartition.getPartition()); } else { return null; } }
/** * This is used only for POST and PUT call to create the pojo. */ public static TopicPartition init(String jsonRequest) { JSONObject jsonObject = JSON.parseObject(jsonRequest); if (!jsonObject.containsKey("topic")) { throw new RuntimeException("Cannot initialize TopicPartitionInfo, missing field: topic"); } if (!jsonObject.containsKey("numPartitions")) { throw new RuntimeException( "Cannot initialize TopicPartitionInfo, missing field: numPartitions"); } return new TopicPartition(jsonObject.getString("topic"), jsonObject.getIntValue("numPartitions")); }
/** * Get stuck topic partitions via offset manager. * * @return the topic partitions that have been stuck for at least _movePartitionAfterStuckMillis. */ private Set<TopicPartition> getStuckTopicPartitions() { Set<TopicPartition> partitions = new HashSet<>(); if (_movePartitionAfterStuckMillis <= 0) { return partitions; } Map<TopicAndPartition, TopicPartitionLag> noProgressMap = _helixMirrorMakerManager.getOffsetMonitor() .getNoProgressTopicToOffsetMap(); long now = System.currentTimeMillis(); for (Map.Entry<TopicAndPartition, TopicPartitionLag> entry : noProgressMap.entrySet()) { TopicPartitionLag lastLag = entry.getValue(); if (now - lastLag.getTimeStamp() > _movePartitionAfterStuckMillis) { partitions.add(new TopicPartition(entry.getKey().topic(), entry.getKey().partition())); } } return partitions; }
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
private void tryAddTopic(String topic) { scala.collection.mutable.Map<String, scala.collection.Map<Object, Seq<Object>>> partitionAssignmentForTopics = _zkUtils.getPartitionAssignmentForTopics(JavaConversions.asScalaBuffer(ImmutableList.of(topic))); if (partitionAssignmentForTopics.get(topic).isEmpty() || partitionAssignmentForTopics.get(topic).get().size() == 0) { LOGGER.info("try to refresh for topic {} but found no topic partition for it", topic); return; } synchronized (_lock) { LOGGER.info("starting to refresh for adding topic {}", topic); if (!getAllTopics().contains(topic)) { try { _topicPartitionInfoMap.put(topic, new TopicPartition(topic, partitionAssignmentForTopics.get(topic).get().size())); } catch (Exception e) { LOGGER.warn("Failed to get topicPartition info for {} from kafka zk: {}", topic, e); } } LOGGER.info("finished refreshing for adding topic {}", topic); } }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); if (liveInstance != null) { customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public static IdealState buildCustomIdealStateFor(String topicName, int numTopicPartitions, PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(numTopicPartitions).setNumReplica(1) .setMaxPartitionsPerNode(numTopicPartitions); for (int i = 0; i < numTopicPartitions; ++i) { synchronized (instanceToNumServingTopicPartitionMap) { InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll(); customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i), liveInstance.getInstanceName(), "ONLINE"); liveInstance.addTopicPartition(new TopicPartition(topicName, i)); instanceToNumServingTopicPartitionMap.add(liveInstance); } } return customModeIdealStateBuilder.build(); }
public Request getTopicExpansionRequestUrl(String topic, int numPartitions) { Request request = new Request(Method.PUT, _baseUrl + "/topics/"); TopicPartition topicPartitionInfo = new TopicPartition(topic, numPartitions); request.setEntity(topicPartitionInfo.toJSON().toJSONString(), MediaType.APPLICATION_JSON); return request; } }
public Request getTopicCreationRequestUrl(String topic, int numPartitions) { Request request = new Request(Method.POST, _baseUrl + "/topics/"); TopicPartition topicPartitionInfo = new TopicPartition(topic, numPartitions); request.setEntity(topicPartitionInfo.toJSON().toJSONString(), MediaType.APPLICATION_JSON); return request; }