/** * Returns the broker instances in the cluster for the given tenant. * * TODO: refactor code to use this method if applicable to reuse instance configs in order to reduce ZK accesses */ public static Set<String> getBrokerInstancesForTenant(List<InstanceConfig> instanceConfigs, String tenant) { return new HashSet<>(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getBrokerTagForTenant(tenant))); } }
public static String getBrokerTagForTenant(String tenantName) { if (tenantName == null) { return TagNameUtils.getBrokerTagForTenant(DEFAULT_TENANT_NAME); } if (tenantName.endsWith(TenantRole.BROKER.toString())) { return tenantName; } else { return TagNameUtils.buildBrokerTenantTagFromTenantName(tenantName); } }
public PinotResourceManagerResponse deleteBrokerTenantFor(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag); for (String instance : instancesInClusterWithTag) { retagInstance(instance, brokerTag, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE); } return PinotResourceManagerResponse.SUCCESS; }
public boolean isBrokerTenantDeletable(String tenantName) { String brokerTag = TagNameUtils.getBrokerTagForTenant(tenantName); Set<String> taggedInstances = new HashSet<>(HelixHelper.getInstancesWithTag(_helixZkManager, brokerTag)); String brokerName = CommonConstants.Helix.BROKER_RESOURCE_INSTANCE; IdealState brokerIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, brokerName); for (String partition : brokerIdealState.getPartitionSet()) { for (String instance : brokerIdealState.getInstanceSet(partition)) { if (taggedInstances.contains(instance)) { return false; } } } return true; }
public boolean isTenantExisted(String tenantName) { if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getBrokerTagForTenant(tenantName)).isEmpty()) { return true; } if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getOfflineTagForTenant(tenantName)).isEmpty()) { return true; } if (!HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getRealtimeTagForTenant(tenantName)).isEmpty()) { return true; } return false; }
public PinotResourceManagerResponse updateBrokerTenant(Tenant tenant) { String brokerTenantTag = TagNameUtils.getBrokerTagForTenant(tenant.getTenantName()); List<String> instancesInClusterWithTag = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTenantTag); if (instancesInClusterWithTag.size() > tenant.getNumberOfInstances()) { return scaleDownBroker(tenant, brokerTenantTag, instancesInClusterWithTag); } if (instancesInClusterWithTag.size() < tenant.getNumberOfInstances()) { return scaleUpBroker(tenant, brokerTenantTag, instancesInClusterWithTag); } return PinotResourceManagerResponse.SUCCESS; }
private void addInstanceTagIfNeeded(String clusterName, String instanceName) { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(clusterName, instanceName); List<String> instanceTags = instanceConfig.getTags(); if (instanceTags == null || instanceTags.isEmpty()) { if (ZKMetadataProvider.getClusterTenantIsolationEnabled(_propertyStore)) { _helixAdmin.addInstanceTag(clusterName, instanceName, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { _helixAdmin.addInstanceTag(clusterName, instanceName, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE); } } }
public PinotResourceManagerResponse createBrokerTenant(Tenant brokerTenant) { List<String> unTaggedInstanceList = getOnlineUnTaggedBrokerInstanceList(); int numberOfInstances = brokerTenant.getNumberOfInstances(); if (unTaggedInstanceList.size() < numberOfInstances) { String message = "Failed to allocate broker instances to Tag : " + brokerTenant.getTenantName() + ", Current number of untagged server instances : " + unTaggedInstanceList.size() + ", Request asked number is : " + brokerTenant.getNumberOfInstances(); LOGGER.error(message); return PinotResourceManagerResponse.failure(message); } String brokerTag = TagNameUtils.getBrokerTagForTenant(brokerTenant.getTenantName()); for (int i = 0; i < brokerTenant.getNumberOfInstances(); ++i) { retagInstance(unTaggedInstanceList.get(i), CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE, brokerTag); } return PinotResourceManagerResponse.SUCCESS; }
public List<String> getBrokerInstancesForTable(String tableName, TableType tableType) { TableConfig tableConfig = getTableConfig(tableName, tableType); String brokerTenantName = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); List<String> serverInstances = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTenantName); return serverInstances; }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
/** * Get all the broker instances for the given table name. * * @param tableName Table name with or without type suffix * @return List of broker instance Ids */ @Nonnull public List<String> getBrokerInstancesFor(@Nonnull String tableName) { String brokerTenantName = null; TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName); if (offlineTableConfig != null) { brokerTenantName = offlineTableConfig.getTenantConfig().getBroker(); } else { TableConfig realtimeTableConfig = ZKMetadataProvider.getRealtimeTableConfig(_propertyStore, tableName); if (realtimeTableConfig != null) { brokerTenantName = realtimeTableConfig.getTenantConfig().getBroker(); } } return HelixHelper.getInstancesWithTag(_helixZkManager, TagNameUtils.getBrokerTagForTenant(brokerTenantName)); }
sendPostRequest(_controllerRequestURLBuilder.forTenantCreate(), payload); Assert.assertEquals( _helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getBrokerTagForTenant(brokerTenant)) .size(), NUM_BROKERS_PER_TAG); Assert.assertEquals( sendPutRequest(_controllerRequestURLBuilder.forTenantCreate(), payload); Assert.assertEquals( _helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getBrokerTagForTenant(brokerTenant)) .size(), i); Assert.assertEquals( sendDeleteRequest(_controllerRequestURLBuilder.forBrokerTenantDelete(brokerTenant)); Assert.assertEquals( _helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getBrokerTagForTenant(brokerTenant)) .size(), 0); Assert.assertEquals(
public static void addFakeBrokerInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception { for (int i = 0; i < numInstances; ++i) { final String brokerId = "Broker_localhost_" + i; final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptyBrokerOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptyBrokerOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, UNTAGGED_BROKER_INSTANCE); } } }
TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getRealtimeTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20);
String brokerTenantName = TagNameUtils.getBrokerTagForTenant(tenantConfig.getBroker()); List<String> brokersForTenant = HelixHelper.getInstancesWithTag(_helixZkManager, brokerTenantName); if (brokersForTenant.isEmpty()) {
helixAdmin.addInstance(HELIX_CLUSTER_NAME, instanceConfig); helixAdmin.addInstanceTag(HELIX_CLUSTER_NAME, instanceConfig.getInstanceName(), TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); idealState = HelixHelper.getBrokerIdealStates(helixAdmin, HELIX_CLUSTER_NAME);
.removeInstanceTag(_helixClusterName, brokerInstance, TagNameUtils.getBrokerTagForTenant(BROKER_TENANT_NAME)); _helixAdmin.addInstanceTag(_helixClusterName, brokerInstance, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE); .removeInstanceTag(_helixClusterName, brokerInstance, TagNameUtils.getBrokerTagForTenant(BROKER_TENANT_NAME)); _helixAdmin.addInstanceTag(_helixClusterName, brokerInstance, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE);