/** * Get all resource names. * * @return List of resource names */ @Nonnull public List<String> getAllResources() { return _helixAdmin.getResourcesInCluster(_helixClusterName); }
private int getNumSegmentLoaded() { InstanceDataManager instanceDataManager = _serverInstance.getInstanceDataManager(); if (instanceDataManager == null) { return -1; } List<String> tableNames = _helixAdmin.getResourcesInCluster(_helixClusterName); int numSegmentsLoaded = 0; for (String tableName : tableNames) { numSegmentsLoaded += instanceDataManager.getAllSegmentsMetadata(tableName).size(); } return numSegmentsLoaded; }
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
public IdealStateMatchServiceStatusCallback(HelixManager helixManager, String clusterName, String instanceName) { _clusterName = clusterName; _instanceName = instanceName; _helixAdmin = helixManager.getClusterManagmentTool(); _helixDataAccessor = helixManager.getHelixDataAccessor(); _resourcesToMonitor = new HashSet<>(); for (String resourceName : _helixAdmin.getResourcesInCluster(_clusterName)) { // Only monitor table resources and broker resource if (!TableNameBuilder.isTableResource(resourceName) && !resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { continue; } // Only monitor enabled resources IdealState idealState = getResourceIdealState(resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceName)) { _resourcesToMonitor.add(resourceName); break; } } } } _numTotalResourcesToMonitor = _resourcesToMonitor.size(); LOGGER.info("Monitoring {} resources: {} for start up of instance {}", _numTotalResourcesToMonitor, _resourcesToMonitor, _instanceName); }
/** * Toggle the status of the table between OFFLINE and ONLINE. * * @param tableName: Name of the table for which to toggle the status. * @param status: True for ONLINE and False for OFFLINE. * @return */ public PinotResourceManagerResponse toggleTableState(String tableName, boolean status) { if (!_helixAdmin.getResourcesInCluster(_helixClusterName).contains(tableName)) { return PinotResourceManagerResponse.failure("Table " + tableName + " not found"); } _helixAdmin.enableResource(_helixClusterName, tableName, status); // If enabling a resource, also reset segments in error state for that resource boolean resetSuccessful = false; if (status) { try { _helixAdmin.resetResource(_helixClusterName, Collections.singletonList(tableName)); resetSuccessful = true; } catch (HelixException e) { LOGGER.warn("Caught exception while resetting resource {}, ignoring.", e, tableName); } } return (status) ? PinotResourceManagerResponse .success("Table " + tableName + " enabled (reset success = " + resetSuccessful + ")") : PinotResourceManagerResponse.success("Table " + tableName + " disabled"); }
/** * Drop the table from helix cluster. * * @param tableName: Name of table to be dropped. * @return */ public PinotResourceManagerResponse dropTable(String tableName) { if (!_helixAdmin.getResourcesInCluster(_helixClusterName).contains(tableName)) { return PinotResourceManagerResponse.failure("Table " + tableName + " not found"); } if (getSegmentsFor(tableName).size() != 0) { return PinotResourceManagerResponse.failure("Table " + tableName + " has segments, drop them first"); } _helixAdmin.dropResource(_helixClusterName, tableName); // remove from property store ZKMetadataProvider.removeResourceSegmentsFromPropertyStore(getPropertyStore(), tableName); ZKMetadataProvider.removeResourceConfigFromPropertyStore(getPropertyStore(), tableName); return PinotResourceManagerResponse.success("Table " + tableName + " dropped"); }
private void logSegmentsLoadingInfo() { InstanceDataManager instanceDataManager = _serverInstance.getInstanceDataManager(); if (instanceDataManager == null) { return; } HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); LiveInstance liveInstance = helixDataAccessor.getProperty(keyBuilder.liveInstance(_instanceId)); String sessionId = liveInstance.getSessionId(); List<String> tableNames = _helixAdmin.getResourcesInCluster(_helixClusterName); for (String tableName : tableNames) { PropertyKey currentStateKey = keyBuilder.currentState(_instanceId, sessionId, tableName); CurrentState currentState = helixDataAccessor.getProperty(currentStateKey); int numSegmentsLoaded = instanceDataManager.getAllSegmentsMetadata(tableName).size(); if (currentState != null && currentState.isValid()) { int numSegmentsToLoad = currentState.getPartitionStateMap().size(); LOGGER.info( "Segments are not fully loaded during server bootstrap, current progress: table: {}, segments loading progress [ {} / {} ]", tableName, numSegmentsLoaded, numSegmentsToLoad); } } }
public void deleteOfflineTable(String tableName) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); // Remove the table from brokerResource HelixHelper.removeResourceFromBrokerIdealState(_helixZkManager, offlineTableName); // Drop the table if (_helixAdmin.getResourcesInCluster(_helixClusterName).contains(offlineTableName)) { _helixAdmin.dropResource(_helixClusterName, offlineTableName); } // Remove all segments for the table _segmentDeletionManager.removeSegmentsFromStore(offlineTableName, getSegmentsFor(offlineTableName)); ZKMetadataProvider.removeResourceSegmentsFromPropertyStore(_propertyStore, offlineTableName); // Remove table config ZKMetadataProvider.removeResourceConfigFromPropertyStore(_propertyStore, offlineTableName); // Remove replica group partition assignment ZKMetadataProvider.removeInstancePartitionAssignmentFromPropertyStore(_propertyStore, offlineTableName); }
private int getNumSegmentsToLoad() { InstanceDataManager instanceDataManager = _serverInstance.getInstanceDataManager(); if (instanceDataManager == null) { return -1; } HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor(); Builder keyBuilder = helixDataAccessor.keyBuilder(); int numSegmentsToLoad = 0; List<String> tableNames = _helixAdmin.getResourcesInCluster(_helixClusterName); for (String tableName : tableNames) { LiveInstance liveInstance = helixDataAccessor.getProperty(keyBuilder.liveInstance(_instanceId)); String sessionId = liveInstance.getSessionId(); PropertyKey currentStateKey = keyBuilder.currentState(_instanceId, sessionId, tableName); CurrentState currentState = helixDataAccessor.getProperty(currentStateKey); if (currentState != null && currentState.isValid()) { numSegmentsToLoad += currentState.getPartitionStateMap().size(); } } return numSegmentsToLoad; }
public void deleteRealtimeTable(String tableName) { String realtimeTableName = TableNameBuilder.REALTIME.tableNameWithType(tableName); // Remove the table from brokerResource HelixHelper.removeResourceFromBrokerIdealState(_helixZkManager, realtimeTableName); // Cache the state and drop the table Set<String> instancesForTable = null; if (_helixAdmin.getResourcesInCluster(_helixClusterName).contains(realtimeTableName)) { instancesForTable = getAllInstancesForTable(realtimeTableName); _helixAdmin.dropResource(_helixClusterName, realtimeTableName); } // Remove all segments for the table _segmentDeletionManager.removeSegmentsFromStore(realtimeTableName, getSegmentsFor(realtimeTableName)); ZKMetadataProvider.removeResourceSegmentsFromPropertyStore(_propertyStore, realtimeTableName); // Remove table config ZKMetadataProvider.removeResourceConfigFromPropertyStore(_propertyStore, realtimeTableName); // Remove groupId/PartitionId mapping for HLC table if (instancesForTable != null) { for (String instance : instancesForTable) { InstanceZKMetadata instanceZKMetadata = ZKMetadataProvider.getInstanceZKMetadata(_propertyStore, instance); if (instanceZKMetadata != null) { instanceZKMetadata.removeResource(realtimeTableName); ZKMetadataProvider.setInstanceZKMetadata(_propertyStore, instanceZKMetadata); } } } }
public List<String> getTopicLists() { return _helixAdmin.getResourcesInCluster(_helixClusterName); }
public boolean isTopicExisted(String topicName) { return _helixAdmin.getResourcesInCluster(_helixClusterName).contains(topicName); }
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) { Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); int numPartitions = is.getNumPartitions(); for (int partition = 0; partition < numPartitions; ++partition) { if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) { TopicPartition tpi = new TopicPartition(topic, partition); unassignedPartitions.add(tpi); } } } return unassignedPartitions; }
/** * From IdealStates. * @param helixManager * @return InstanceToNumTopicPartitionMap */ public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap( HelixManager helixManager) { Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap = new HashMap<String, Set<TopicPartition>>(); HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); String helixClusterName = helixManager.getClusterName(); for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) { IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic); for (String partition : is.getPartitionSet()) { TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition)); for (String instance : is.getInstanceSet(partition)) { if (!instanceToNumTopicPartitionMap.containsKey(instance)) { instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>()); } instanceToNumTopicPartitionMap.get(instance).add(tpi); } } } return instanceToNumTopicPartitionMap; }
public boolean isPipelineExisted(String pipeline) { return _helixAdmin.getResourcesInCluster(_helixClusterName).contains(pipeline); }
public boolean isTopicExisted(String topicName) { return _helixAdmin.getResourcesInCluster(_helixClusterName).contains(topicName); }
public List<String> getPipelineLists() { List<String> pipelineList = new ArrayList<>(); for (String resource : _helixAdmin.getResourcesInCluster(_helixClusterName)) { if (resource.startsWith(SEPARATOR)) { pipelineList.add(resource); } } return pipelineList; }
public List<String> getTopicLists() { List<String> toplicList = new ArrayList<>(); for (String resource : _helixAdmin.getResourcesInCluster(_helixClusterName)) { if (!resource.startsWith(SEPARATOR)) { toplicList.add(resource); } } return toplicList; }
public void expandCluster(String clusterName) { List<String> resources = _admin.getResourcesInCluster(clusterName); for (String resourceName : resources) { expandResource(clusterName, resourceName); } }
public void expandCluster(String clusterName) { List<String> resources = _admin.getResourcesInCluster(clusterName); for (String resourceName : resources) { expandResource(clusterName, resourceName); } }