/** * Gets the ideal state of the table * @param tableNameWithType Table name with suffix * @return IdealState of tableNameWithType */ public IdealState getTableIdealState(String tableNameWithType) { return _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); }
/** * Tag Pinot Minion instance if needed. */ private void addInstanceTagIfNeeded() { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(_helixClusterName, _instanceId); if (instanceConfig.getTags().isEmpty()) { LOGGER.info("Adding default Helix tag: {} to Pinot minion", CommonConstants.Minion.UNTAGGED_INSTANCE); _helixAdmin.addInstanceTag(_helixClusterName, _instanceId, CommonConstants.Minion.UNTAGGED_INSTANCE); } } }
/** * Check if IdealState = ExternalView. If its not equal, return the number of differing segments. */ public int isStable(String tableName) { IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); ExternalView externalView = _helixAdmin.getResourceExternalView(_helixClusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.debug("Mismatch: segment" + segment + " server:" + server + " state:" + state); numDiff = numDiff + 1; } } } return numDiff; }
private void retagInstance(String instanceName, String oldTag, String newTag) { _helixAdmin.removeInstanceTag(_helixClusterName, instanceName, oldTag); _helixAdmin.addInstanceTag(_helixClusterName, instanceName, newTag); }
/** * Toggle the status of the table between OFFLINE and ONLINE. * * @param tableName: Name of the table for which to toggle the status. * @param status: True for ONLINE and False for OFFLINE. * @return */ public PinotResourceManagerResponse toggleTableState(String tableName, boolean status) { if (!_helixAdmin.getResourcesInCluster(_helixClusterName).contains(tableName)) { return PinotResourceManagerResponse.failure("Table " + tableName + " not found"); } _helixAdmin.enableResource(_helixClusterName, tableName, status); // If enabling a resource, also reset segments in error state for that resource boolean resetSuccessful = false; if (status) { try { _helixAdmin.resetResource(_helixClusterName, Collections.singletonList(tableName)); resetSuccessful = true; } catch (HelixException e) { LOGGER.warn("Caught exception while resetting resource {}, ignoring.", e, tableName); } } return (status) ? PinotResourceManagerResponse .success("Table " + tableName + " enabled (reset success = " + resetSuccessful + ")") : PinotResourceManagerResponse.success("Table " + tableName + " disabled"); }
PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL; if (admin.getClusters().contains(helixClusterName)) { LOGGER.info("cluster already exists ********************************************* "); if (isUpdateStateModel) { final StateModelDefinition curStateModelDef = admin.getStateModelDef(helixClusterName, segmentStateModelName); List<String> states = curStateModelDef.getStatesPriorityList(); if (states.contains(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) { admin.addCluster(helixClusterName, false); admin.setConfig(scope, props); admin.addStateModelDef(helixClusterName, segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition()); + " ********************************************** "); admin.addStateModelDef(helixClusterName, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition()); IdealState idealState = PinotTableIdealStateBuilder .buildEmptyIdealStateForBrokerResource(admin, helixClusterName, enableBatchMessageMode); admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState); initPropertyStorePath(helixClusterName, zkPath); LOGGER.info("New Cluster setup completed... ********************************************** ");
public static void setup() { admin = new ZKHelixAdmin(ZK_ADDRESS); // create cluster echo("Creating cluster: " + CLUSTER_NAME); admin.addCluster(CLUSTER_NAME, true); // Add nodes to the cluster echo("Adding " + NUM_NODES + " participants to the cluster"); for (int i = 0; i < NUM_NODES; i++) { admin.addInstance(CLUSTER_NAME, INSTANCE_CONFIG_LIST.get(i)); echo("\t Added participant: " + INSTANCE_CONFIG_LIST.get(i).getInstanceName()); } // Add a state model StateModelDefinition myStateModel = defineStateModel(); echo("Configuring StateModel: " + "MyStateModel with 1 Master and 1 Slave"); admin.addStateModelDef(CLUSTER_NAME, STATE_MODEL_NAME, myStateModel); // Add a resource with 6 partitions and 2 replicas echo("Adding a resource MyResource: " + "with 6 partitions and 2 replicas"); admin.addResource(CLUSTER_NAME, RESOURCE_NAME, NUM_PARTITIONS, STATE_MODEL_NAME, "AUTO"); // this will set up the ideal state, it calculates the preference list for // each partition similar to consistent hashing admin.rebalance(CLUSTER_NAME, RESOURCE_NAME, NUM_REPLICAS); }
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath) { final HelixAdmin admin = new ZKHelixAdmin(zkPath); if (admin.getClusters().contains(helixClusterName)) { LOGGER.info( "cluster already exist, skipping it.. ********************************************* "); return; } LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName + " was not found ********************************************* "); admin.addCluster(helixClusterName, false); LOGGER.info("Enable mirror maker machines auto join."); final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER) .forCluster(helixClusterName).build(); final Map<String, String> props = new HashMap<String, String>(); props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true)); props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(100)); admin.setConfig(scope, props); LOGGER.info("Adding state model definition named : OnlineOffline generated using : " + OnlineOfflineStateModel.class.toString() + " ********************************************** "); // add state model definition admin.addStateModelDef(helixClusterName, "OnlineOffline", OnlineOfflineStateModel.build()); LOGGER.info("New Cluster setup completed... ********************************************** "); }
IdealState idealState; Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6); idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT); _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); Assert.assertEquals(externalView.getStateMap(DINING_TABLE_NAME).size(), SEGMENT_COUNT); _pinotResourceManager.addTable(tableConfig); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6); idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); Assert.assertEquals(idealState.getInstanceSet(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT); Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT); _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); Assert.assertEquals(externalView.getStateMap(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT); externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME); Assert.assertEquals(externalView.getPartitionSet().size(), SEGMENT_COUNT); tableArray = brokerRoutingTableBuilderMap.keySet().toArray();
public void deleteOfflineTable(String tableName) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(tableName); // Remove the table from brokerResource HelixHelper.removeResourceFromBrokerIdealState(_helixZkManager, offlineTableName); // Drop the table if (_helixAdmin.getResourcesInCluster(_helixClusterName).contains(offlineTableName)) { _helixAdmin.dropResource(_helixClusterName, offlineTableName); } // Remove all segments for the table _segmentDeletionManager.removeSegmentsFromStore(offlineTableName, getSegmentsFor(offlineTableName)); ZKMetadataProvider.removeResourceSegmentsFromPropertyStore(_propertyStore, offlineTableName); // Remove table config ZKMetadataProvider.removeResourceConfigFromPropertyStore(_propertyStore, offlineTableName); // Remove replica group partition assignment ZKMetadataProvider.removeInstancePartitionAssignmentFromPropertyStore(_propertyStore, offlineTableName); }
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
/** * Gets the external view of the table * @param tableNameWithType Table name with suffix * @return ExternalView of tableNameWithType */ public ExternalView getTableExternalView(String tableNameWithType) { return _helixAdmin.getResourceExternalView(_helixClusterName, tableNameWithType); }
@Test public void testEnableDisablePartitions() { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; String instanceName = "TestInstance"; String testResourcePrefix = "TestResource"; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); HelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); admin.addInstance(clusterName, new InstanceConfig(instanceName)); // Test disable instances with resources admin.enablePartition(false, clusterName, instanceName, testResourcePrefix + "0", Arrays.asList(new String[]{"1", "2"})); admin.enablePartition(false, clusterName, instanceName, testResourcePrefix + "1", Arrays.asList(new String[]{"2", "3", "4"})); InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, instanceName); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "0").size(), 2); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "1").size(), 3); // Test disable partition across resources // TODO : Remove this part once setInstanceEnabledForPartition(partition, enabled) is removed instanceConfig.setInstanceEnabledForPartition("10", false); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "0").size(), 3); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "1").size(), 4); admin.dropCluster(clusterName); }
@Test public void testDisableResource() { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); HelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient), "Cluster should be setup"); String resourceName = "TestDB"; admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition( StateModelConfigGenerator.generateConfigForMasterSlave())); admin.addResource(clusterName, resourceName, 4, "MasterSlave"); admin.enableResource(clusterName, resourceName, false); BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient); HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resourceName)); Assert.assertFalse(idealState.isEnabled()); admin.enableResource(clusterName, resourceName, true); idealState = accessor.getProperty(keyBuilder.idealStates(resourceName)); Assert.assertTrue(idealState.isEnabled()); admin.dropCluster(clusterName); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableJSONConfigString); Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getPartitionSet().size(), 1); Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getInstanceSet(tableName + "_OFFLINE").size(), 20); .assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i); _helixResourceManager.addNewSegment(SegmentMetadataMockUtils.mockSegmentMetadata(tableName), "downloadUrl"); Assert .assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i + 1); _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getPartitionSet().size(), 0); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getRealtimeTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(_helixClusterName, TagNameUtils.getOfflineTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)).size(), 20);
public static void updateResourceConfigsFor(Map<String, String> newConfigs, String resourceName, String clusterName, HelixAdmin admin) { final HelixConfigScope scope = getResourceScopeFor(clusterName, resourceName); admin.setConfig(scope, newConfigs); }
private void addInstanceToBrokerIdealState(String brokerTenantTag, String instanceName) { IdealState tableIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE); for (String tableNameWithType : tableIdealState.getPartitionSet()) { TableConfig tableConfig = ZKMetadataProvider.getTableConfig(_propertyStore, tableNameWithType); Preconditions.checkNotNull(tableConfig); String brokerTag = TagNameUtils.getBrokerTagForTenant(tableConfig.getTenantConfig().getBroker()); if (brokerTag.equals(brokerTenantTag)) { tableIdealState.setPartitionState(tableNameWithType, instanceName, BrokerOnlineOfflineStateModel.ONLINE); } } _helixAdmin .setResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, tableIdealState); }
public static void addFakeDataInstanceToAutoJoinHelixCluster(String helixClusterName, String zkServer, String instanceId, boolean isSingleTenant, int adminPort) throws Exception { final HelixManager helixZkManager = HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer); final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine(); final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory(); stateMachineEngine .registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(), stateModelFactory); helixZkManager.connect(); if (isSingleTenant) { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE); } HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.PARTICIPANT, helixClusterName) .forParticipant(instanceId).build(); Map<String, String> props = new HashMap<>(); props.put(CommonConstants.Helix.Instance.ADMIN_PORT_KEY, String.valueOf(adminPort)); helixZkManager.getClusterManagmentTool().setConfig(scope, props); }
/** * Get all resource names. * * @return List of resource names */ @Nonnull public List<String> getAllResources() { return _helixAdmin.getResourcesInCluster(_helixClusterName); }
private void createHelixEntriesForHighLevelConsumer(TableConfig config, String realtimeTableName, IdealState idealState) { if (idealState == null) { idealState = PinotTableIdealStateBuilder .buildInitialHighLevelRealtimeIdealStateFor(realtimeTableName, config, _helixZkManager, _propertyStore, _enableBatchMessageMode); LOGGER.info("Adding helix resource with empty HLC IdealState for {}", realtimeTableName); _helixAdmin.addResource(_helixClusterName, realtimeTableName, idealState); } else { // TODO jfim: We get in this block if we're trying to add a HLC or it already exists. If it doesn't already exist, we need to set instance configs properly (which is done in buildInitialHighLevelRealtimeIdealState, surprisingly enough). For now, do nothing. LOGGER.info("Not reconfiguring HLC for table {}", realtimeTableName); } LOGGER.info("Successfully created empty ideal state for high level consumer for {} ", realtimeTableName); // Finally, create the propertystore entry that will trigger watchers to create segments String tablePropertyStorePath = ZKMetadataProvider.constructPropertyStorePathForResource(realtimeTableName); if (!_propertyStore.exists(tablePropertyStorePath, AccessOption.PERSISTENT)) { _propertyStore.create(tablePropertyStorePath, new ZNRecord(realtimeTableName), AccessOption.PERSISTENT); } }