/** * Returns all instances for the given resource. * * @param idealState IdealState of the resource for which to return the instances of. * @return Returns a Set of strings containing the instance names for the given cluster. */ public static Set<String> getAllInstancesForResource(IdealState idealState) { final Set<String> instances = new HashSet<String>(); for (final String partition : idealState.getPartitionSet()) { for (final String instance : idealState.getInstanceSet(partition)) { instances.add(instance); } } return instances; }
private Set<String> getAllInstancesForTable(String tableName) { Set<String> instanceSet = new HashSet<String>(); IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); for (String partition : tableIdealState.getPartitionSet()) { instanceSet.addAll(tableIdealState.getInstanceSet(partition)); } return instanceSet; }
public boolean isBrokerTenantDeletable(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); Set<String> taggedInstances = new HashSet<>(HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag)); String brokerName = CommonConstants.Helix.BROKER_RESOURCE_INSTANCE; IdealState brokerIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, brokerName); for (String partition : brokerIdealState.getPartitionSet()) { for (String instance : brokerIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } return true; }
if (onlineSegments != null) { for (String segment : onlineSegments) { Set<String> oldInstances = idealState.getInstanceSet(segment); Preconditions.checkArgument(CollectionUtils.isNotEmpty(oldInstances)); for (String instance : oldInstances) {
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
public IdealStateMatchServiceStatusCallback(HelixManager helixManager, String clusterName, String instanceName) { _clusterName = clusterName; _instanceName = instanceName; _helixAdmin = helixManager.getClusterManagmentTool(); _helixDataAccessor = helixManager.getHelixDataAccessor(); _resourcesToMonitor = new HashSet<>(); for (String resourceName : _helixAdmin.getResourcesInCluster(_clusterName)) { // Only monitor table resources and broker resource if (!TableNameBuilder.isTableResource(resourceName) && !resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { continue; } // Only monitor enabled resources IdealState idealState = getResourceIdealState(resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceName)) { _resourcesToMonitor.add(resourceName); break; } } } } _numTotalResourcesToMonitor = _resourcesToMonitor.size(); LOGGER.info("Monitoring {} resources: {} for start up of instance {}", _numTotalResourcesToMonitor, _resourcesToMonitor, _instanceName); }
Set<String> currentSegmentInstances = idealState.getInstanceSet(currentSegmentId); for (String instance : currentSegmentInstances) { idealState.setPartitionState(currentSegmentId, instance,
private Map<String, Integer> getSegmentsPerInstance(String tableName) { Map<String, Integer> segmentsPerInstance = new HashMap<String, Integer>(); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableName + "_OFFLINE"); for (String partitionName : idealState.getPartitionSet()) { for (String instanceName : idealState.getInstanceSet(partitionName)) { if (!segmentsPerInstance.containsKey(instanceName)) { segmentsPerInstance.put(instanceName, 1); } else { segmentsPerInstance.put(instanceName, segmentsPerInstance.get(instanceName) + 1); } } } return segmentsPerInstance; }
public PinotResourceManagerResponse rebuildBrokerResource(String tableNameWithType, Set<String> brokerInstances) { IdealState brokerIdealState = HelixHelper.getBrokerIdealStates(_helixAdmin, _helixClusterName); Set<String> brokerInstancesInIdealState = brokerIdealState.getInstanceSet(tableNameWithType); if (brokerInstancesInIdealState.equals(brokerInstances)) { return PinotResourceManagerResponse .success("Broker resource is not rebuilt because ideal state is the same for table: " + tableNameWithType); } // Update ideal state with the new broker instances try { HelixHelper.updateIdealState(getHelixZkManager(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState -> { assert idealState != null; Map<String, String> instanceStateMap = idealState.getInstanceStateMap(tableNameWithType); if (instanceStateMap != null) { instanceStateMap.clear(); } for (String brokerInstance : brokerInstances) { idealState.setPartitionState(tableNameWithType, brokerInstance, BrokerOnlineOfflineStateModel.ONLINE); } return idealState; }, DEFAULT_RETRY_POLICY); LOGGER.info("Successfully rebuilt brokerResource for table: {}", tableNameWithType); return PinotResourceManagerResponse.success("Rebuilt brokerResource for table: " + tableNameWithType); } catch (Exception e) { LOGGER.error("Caught exception while rebuilding broker resource for table: {}", tableNameWithType, e); throw e; } }
public boolean isServerTenantDeletable(String tenantName) { Set<String> taggedInstances = new HashSet<>( HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getOfflineTagForTenant(tenantName))); taggedInstances .addAll(HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getRealtimeTagForTenant(tenantName))); for (String resourceName : getAllResources()) { if (!TableNameBuilder.isTableResource(resourceName)) { continue; } IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, resourceName); for (String partition : tableIdealState.getPartitionSet()) { for (String instance : tableIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } } return true; }
do { final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); final Set<String> instanceSet = idealState.getInstanceSet(segmentName); if (instanceSet == null || instanceSet.size() == 0) { final Set<String> instanceSet = idealState.getInstanceSet(segmentName); LOGGER.info("Found {} instances for segment '{}', in ideal state", instanceSet.size(), segmentName); for (final String instance : instanceSet) {
final Set<String> instanceSet = idealState.getInstanceSet(segmentName); if (instanceSet == null || instanceSet.isEmpty()) { return PinotResourceManagerResponse.failure("Segment " + segmentName + " not found");
private void testCommitSegmentEntries(FakePinotLLCRealtimeSegmentManager segmentManager, LLCRealtimeSegmentZKMetadata committingSegmentMetadata, ZNRecord oldZnRec, ZNRecord newZnRec, Set<String> prevInstances, PartitionAssignment partitionAssignment, int partition) { // Get the old and new segment metadata and make sure that they are correct. LLCRealtimeSegmentZKMetadata oldMetadata = new LLCRealtimeSegmentZKMetadata(oldZnRec); LLCRealtimeSegmentZKMetadata newMetadata = new LLCRealtimeSegmentZKMetadata(newZnRec); LLCSegmentName oldSegmentName = new LLCSegmentName(oldMetadata.getSegmentName()); LLCSegmentName newSegmentName = new LLCSegmentName(newMetadata.getSegmentName()); // Assert on propertystore entries Assert.assertEquals(oldSegmentName.getSegmentName(), committingSegmentMetadata.getSegmentName()); Assert.assertEquals(newSegmentName.getPartitionId(), oldSegmentName.getPartitionId()); Assert.assertEquals(newSegmentName.getSequenceNumber(), oldSegmentName.getSequenceNumber() + 1); Assert.assertEquals(oldMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.DONE); Assert.assertEquals(newMetadata.getStatus(), CommonConstants.Segment.Realtime.Status.IN_PROGRESS); Assert.assertNotNull(oldMetadata.getDownloadUrl()); Assert.assertEquals(Long.valueOf(oldMetadata.getCrc()), Long.valueOf(FakePinotLLCRealtimeSegmentManager.CRC)); Assert.assertEquals(oldMetadata.getStartTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getStartMillis()); Assert.assertEquals(oldMetadata.getEndTime(), FakePinotLLCRealtimeSegmentManager.INTERVAL.getEndMillis()); Assert.assertEquals(oldMetadata.getTotalRawDocs(), FakePinotLLCRealtimeSegmentManager.NUM_DOCS); Assert.assertEquals(oldMetadata.getIndexVersion(), FakePinotLLCRealtimeSegmentManager.SEGMENT_VERSION); // check ideal state IdealState idealState = segmentManager._tableIdealState; Set<String> currInstances = idealState.getInstanceSet(oldSegmentName.getSegmentName()); Assert.assertEquals(prevInstances.size(), currInstances.size()); Assert.assertTrue(prevInstances.containsAll(currInstances)); Set<String> newSegmentInstances = idealState.getInstanceSet(newSegmentName.getSegmentName()); List<String> expectedNewInstances = partitionAssignment.getInstancesListForPartition(String.valueOf(partition)); Assert.assertEquals(newSegmentInstances.size(), expectedNewInstances.size()); Assert.assertTrue(newSegmentInstances.containsAll(expectedNewInstances)); }
Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT); Assert.assertEquals(idealState.getInstanceSet(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT); Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getInstanceSet(tableName + "_OFFLINE").size(), 20);
Assert.assertTrue(idealState.getInstanceSet(partitionName) .equals(_pinotHelixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME))); _pinotHelixResourceManager.rebuildBrokerResourceFromHelixTags(partitionName); idealState = HelixHelper.getBrokerIdealStates(helixAdmin, HELIX_CLUSTER_NAME); Assert.assertTrue(!idealState.getInstanceSet(partitionNameTwo) .equals(_pinotHelixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME))); _pinotHelixResourceManager.rebuildBrokerResourceFromHelixTags(partitionNameTwo); idealState = HelixHelper.getBrokerIdealStates(helixAdmin, HELIX_CLUSTER_NAME); Assert.assertTrue(idealState.getInstanceSet(partitionNameTwo) .equals(_pinotHelixResourceManager.getAllInstancesForBrokerTenant(TagNameUtils.DEFAULT_TENANT_NAME)));
IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, resource); for (String partition : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partition).contains(instanceName)) { return PinotResourceManagerResponse .failure("Instance " + instanceName + " exists in ideal state for " + resource);
segmentManager._paths.clear(); segmentManager._records.clear(); Set<String> prevInstances = idealState.getInstanceSet(committingSegmentMetadata.getSegmentName()); reqParams.withSegmentName(committingSegmentMetadata.getSegmentName()).withOffset(nextOffset); CommittingSegmentDescriptor committingSegmentDescriptor = segmentManager._paths.clear(); segmentManager._records.clear(); prevInstances = idealState.getInstanceSet(committingSegmentMetadata.getSegmentName()); reqParams.withSegmentName(committingSegmentMetadata.getSegmentName()).withOffset(nextOffset); committingSegmentDescriptor = CommittingSegmentDescriptor.fromSegmentCompletionReqParams(reqParams); segmentManager._records.clear(); committingSegmentMetadata = new LLCRealtimeSegmentZKMetadata(newZnRec); prevInstances = idealState.getInstanceSet(committingSegmentMetadata.getSegmentName()); reqParams.withSegmentName(committingSegmentMetadata.getSegmentName()).withOffset(nextOffset); committingSegmentDescriptor = CommittingSegmentDescriptor.fromSegmentCompletionReqParams(reqParams);
Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getInstanceSet(OFFLINE_TABLE_NAME).size(), NUM_INSTANCES);
for (String server : rebalancedIdealState.getInstanceSet(segment)) { instances.remove(server);