public AutoAddInvertedIndex(@Nonnull String zkAddress, @Nonnull String clusterName, @Nonnull String controllerAddress, @Nonnull String brokerAddress, @Nonnull Strategy strategy, @Nonnull Mode mode) { _clusterName = clusterName; _controllerAddress = controllerAddress; _brokerAddress = brokerAddress; _helixAdmin = new ZKHelixAdmin(zkAddress); _propertyStore = new ZkHelixPropertyStore<>(zkAddress, new ZNRecordSerializer(), PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, clusterName)); _strategy = strategy; _mode = mode; }
clusterInfo.clusterName = _clusterName; ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(_zkAddress); if (!zkHelixAdmin.getClusters().contains(_clusterName)) { LOGGER.error("Cluster {} not found in {}.", _clusterName, _zkAddress); return false; List<String> instancesInCluster = zkHelixAdmin.getInstancesInCluster(_clusterName); List<String> tables = zkHelixAdmin.getResourcesInCluster(_clusterName); serverInfo.name = server; serverInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE"; InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server); serverInfo.tags = config.getRecord().getListField("TAG_LIST"); clusterInfo.addServerInfo(serverInfo); brokerInfo.name = server; brokerInfo.state = (liveInstances.contains(server)) ? "ONLINE" : "OFFLINE"; InstanceConfig config = zkHelixAdmin.getInstanceConfig(_clusterName, server); brokerInfo.tags = config.getRecord().getListField("TAG_LIST"); clusterInfo.addBrokerInfo(brokerInfo); IdealState idealState = zkHelixAdmin.getResourceIdealState(_clusterName, table); ExternalView externalView = zkHelixAdmin.getResourceExternalView(_clusterName, table); Set<String> segmentsFromIdealState = idealState.getPartitionSet();
/** * return true if IdealState = ExternalView * @return */ public int isStable(String tableName) { IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName); ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.info( "Mismatch: segment " + segment + " server:" + server + " expected state:" + state + " actual state:" + ( (mapEV == null || mapEV.get(server) == null) ? "null" : mapEV.get(server))); numDiff = numDiff + 1; } } } return numDiff; }
public void fixTableIdealState(String tableName) throws Exception { IdealState idealState = _helixAdmin.getResourceIdealState(_clusterName, tableName); if (idealState == null) { LOGGER.info("No IDEALSTATE found for table " + tableName); if (_fix) { LOGGER.info("Replacing IDEALSTATE for table " + tableName + " with " + nChanges + " changes"); _helixAdmin.setResourceIdealState(_clusterName, tableName, idealState); } else { LOGGER.info("Detected " + nChanges + " instances in " + fromState + " in table " + tableName);
@Override public boolean execute() throws Exception { ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress); List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName); IdealState resourceIdealState = helixAdmin.getResourceIdealState(clusterName, resourceName); ExternalView resourceExternalView = helixAdmin.getResourceExternalView(clusterName, resourceName); Map<String, Map<String, String>> mapFieldsFromIS = resourceIdealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsFromEV = resourceExternalView.getRecord().getMapFields();
new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer()); ZKHelixAdmin admin = new ZKHelixAdmin(zkclient); admin.addCluster(clusterName, true); admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL, new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline())); admin.addResource(clusterName, resourceName, DEFAULT_PARTITION_NUMBER, DEFAULT_STATE_MODEL, RebalanceMode.FULL_AUTO.toString()); admin.rebalance(clusterName, resourceName, 1);
ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient); HelixConfigScope participantScope = admin.addInstance(clusterName, new InstanceConfig("localhost_12918"));
/** * Configure the instance, the configuration of each node is available to * other nodes. * @param instanceName */ private void configureInstance(String instanceName) { ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress); List<String> instancesInCluster = helixAdmin.getInstancesInCluster(clusterName); if (instancesInCluster == null || !instancesInCluster.contains(instanceName)) { InstanceConfig config = new InstanceConfig(instanceName); config.setHostName("localhost"); config.setPort("12000"); helixAdmin.addInstance(clusterName, config); } }
public static void waitForExternalViewUpdate(String zkAddress, final String clusterName, long timeoutInMilliseconds) { final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress); List<String> allResourcesInCluster = helixAdmin.getResourcesInCluster(clusterName); Set<String> tableAndBrokerResources = new HashSet<>(); for (String resourceName : allResourcesInCluster) { // Only check table resources and broker resource if (TableNameBuilder.isTableResource(resourceName) || resourceName .equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) { tableAndBrokerResources.add(resourceName); } } StrictMatchExternalViewVerifier verifier = new StrictMatchExternalViewVerifier.Builder(clusterName).setZkAddr(zkAddress) .setResources(tableAndBrokerResources).build(); boolean success = verifier.verify(timeoutInMilliseconds); if (success) { LOGGER.info("Cluster is ready to serve queries"); } }
ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient); ConfigScope participantScope = admin.addInstance(clusterName, new InstanceConfig("localhost_12918")); Assert.assertEquals(participantConfigValue, "participantConfigValue"); admin.dropCluster(clusterName); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
private static void addInstanceConfig(String instanceName) { // add node to cluster if not already added ZKHelixAdmin admin = new ZKHelixAdmin(ZK_ADDR); InstanceConfig instanceConfig = null; try { instanceConfig = admin.getInstanceConfig(clusterName, instanceName); } catch (Exception e) { } if (instanceConfig == null) { InstanceConfig config = new InstanceConfig(instanceName); config.setHostName("localhost"); config.setInstanceEnabled(true); echo("Adding InstanceConfig:" + config); admin.addInstance(clusterName, config); } }
System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis())); _admin = new ZKHelixAdmin(_gZkClient); String namespace = "/" + CLUSTER_NAME; if (_gZkClient.exists(namespace)) { IdealState.RebalanceMode.FULL_AUTO + ""); IdealState idealState = _admin.getResourceIdealState(CLUSTER_NAME, TEST_DB1); idealState.setDisableExternalView(true); _admin.setResourceIdealState(CLUSTER_NAME, TEST_DB1, idealState);
public static void setupEmptyCluster(HelixZkClient zkClient, String clusterName) { ZKHelixAdmin admin = new ZKHelixAdmin(zkClient); admin.addCluster(clusterName, true); }
ZKHelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.enableCluster(clusterName, false); admin.dropResource(clusterName, "TestDB0"); admin.enableCluster(clusterName, true);
ZKHelixAdmin tool = new ZKHelixAdmin(_gZkClient); tool.enablePartition(false, clusterName, "localhost_12918", "TestDB0", Arrays.asList("TestDB0_4")); tool.enableInstance(clusterName, "localhost_12918", false); tool.enablePartition(true, clusterName, "localhost_12918", "TestDB0", Arrays.asList("TestDB0_4")); tool.enableInstance(clusterName, "localhost_12918", true);
@Test() public void testDisableNode() throws Exception { String command = "-zkSvr " + ZK_ADDR + " -enableInstance " + CLUSTER_NAME + " " + PARTICIPANT_PREFIX + "_12918" + " TestDB TestDB_0 false"; ClusterSetup.processCommandLineArgs(command.split(" ")); boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier( ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); ZKHelixAdmin tool = new ZKHelixAdmin(_gZkClient); tool.enableInstance(CLUSTER_NAME, PARTICIPANT_PREFIX + "_12918", true); result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier( ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); } }
@Test() public void testDisablePartition() throws Exception { LOG.info("START testDisablePartition() at " + new Date(System.currentTimeMillis())); // localhost_12919 is MASTER for TestDB_0 String command = "--zkSvr " + ZK_ADDR + " --enablePartition false " + CLUSTER_NAME + " localhost_12919 TestDB TestDB_0 TestDB_9"; ClusterSetup.processCommandLineArgs(command.split("\\s+")); Map<String, Set<String>> map = new HashMap<String, Set<String>>(); map.put("TestDB_0", TestHelper.setOf("localhost_12919")); map.put("TestDB_9", TestHelper.setOf("localhost_12919")); boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier( ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); TestHelper.verifyState(CLUSTER_NAME, ZK_ADDR, map, "OFFLINE"); ZKHelixAdmin tool = new ZKHelixAdmin(_gZkClient); tool.enablePartition(true, CLUSTER_NAME, "localhost_12919", "TestDB", Arrays.asList("TestDB_9")); result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier( ZK_ADDR, CLUSTER_NAME)); Assert.assertTrue(result); map.clear(); map.put("TestDB_0", TestHelper.setOf("localhost_12919")); TestHelper.verifyState(CLUSTER_NAME, ZK_ADDR, map, "OFFLINE"); map.clear(); map.put("TestDB_9", TestHelper.setOf("localhost_12919")); TestHelper.verifyState(CLUSTER_NAME, ZK_ADDR, map, "MASTER"); LOG.info("STOP testDisablePartition() at " + new Date(System.currentTimeMillis())); }
@Override public void rebalance(String clusterName, String resourceName, int replica, String keyPrefix, String group) { List<String> instanceNames = new LinkedList<String>(); if (keyPrefix == null || keyPrefix.length() == 0) { keyPrefix = resourceName; } if (group != null && group.length() > 0) { instanceNames = getInstancesInClusterWithTag(clusterName, group); } if (instanceNames.size() == 0) { logger.info("No tags found for resource " + resourceName + ", use all instances"); instanceNames = getInstancesInCluster(clusterName); group = ""; } else { logger.info("Found instances with tag for " + resourceName + " " + instanceNames); } rebalance(clusterName, resourceName, replica, keyPrefix, instanceNames, group); }
@Override public void addResource(String clusterName, String resourceName, int partitions, String stateModelRef, String rebalancerMode, int bucketSize) { addResource(clusterName, resourceName, partitions, stateModelRef, rebalancerMode, bucketSize, -1); }
IdealState idealState = getResourceIdealState(clusterName, resourceName); if (idealState == null) { throw new HelixException("Resource: " + resourceName + " has NOT been added yet"); int partitions = idealState.getNumPartitions(); String stateModelName = idealState.getStateModelDefRef(); StateModelDefinition stateModDef = getStateModelDef(clusterName, stateModelName); setResourceIdealState(clusterName, resourceName, idealState);