/** * Tag Pinot Minion instance if needed. */ private void addInstanceTagIfNeeded() { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(_helixClusterName, _instanceId); if (instanceConfig.getTags().isEmpty()) { LOGGER.info("Adding default Helix tag: {} to Pinot minion", CommonConstants.Minion.UNTAGGED_INSTANCE); _helixAdmin.addInstanceTag(_helixClusterName, _instanceId, CommonConstants.Minion.UNTAGGED_INSTANCE); } } }
private void addInstanceTagIfNeeded(String clusterName, String instanceName) { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(clusterName, instanceName); List<String> instanceTags = instanceConfig.getTags(); if (instanceTags == null || instanceTags.isEmpty()) { if (ZKMetadataProvider.getClusterTenantIsolationEnabled(_propertyStore)) { _helixAdmin.addInstanceTag(clusterName, instanceName, TagNameUtils.getBrokerTagForTenant(TagNameUtils.DEFAULT_TENANT_NAME)); } else { _helixAdmin.addInstanceTag(clusterName, instanceName, CommonConstants.Helix.UNTAGGED_BROKER_INSTANCE); } } }
/** * Helix participant cannot pre-configure tags before it connects to ZK. So this method can only be invoked after * {@link HelixManager#connect()}. However this will still work because tagged jobs won't be sent to a non-tagged instance. Hence * the job with EXAMPLE_INSTANCE_TAG will remain in the ZK until an instance with EXAMPLE_INSTANCE_TAG was found. */ private void addInstanceTags() { List<String> tags = ConfigUtils.getStringList(this.config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY); HelixManager receiverManager = getReceiverManager(); if (receiverManager.isConnected()) { if (!tags.isEmpty()) { logger.info("Adding tags binding " + tags); tags.forEach(tag -> receiverManager.getClusterManagmentTool() .addInstanceTag(this.clusterName, this.helixInstanceName, tag)); logger.info("Actual tags binding " + receiverManager.getClusterManagmentTool() .getInstanceConfig(this.clusterName, this.helixInstanceName).getTags()); } } }
private void addInstanceTagIfNeeded(String clusterName, String instanceName) { InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(clusterName, instanceName); List<String> instanceTags = instanceConfig.getTags(); if (instanceTags == null || instanceTags.size() == 0) { if (ZKMetadataProvider.getClusterTenantIsolationEnabled(_helixManager.getHelixPropertyStore())) { _helixAdmin.addInstanceTag(clusterName, instanceName, TableNameBuilder.OFFLINE.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); _helixAdmin.addInstanceTag(clusterName, instanceName, TableNameBuilder.REALTIME.tableNameWithType(TagNameUtils.DEFAULT_TENANT_NAME)); } else { _helixAdmin.addInstanceTag(clusterName, instanceName, CommonConstants.Helix.UNTAGGED_SERVER_INSTANCE); } } }
@Test public void testGetInstanceConfigs() throws Exception { Set<String> servers = _helixResourceManager.getAllInstancesForServerTenant(SERVER_TENANT_NAME); for (String server : servers) { InstanceConfig cachedInstanceConfig = _helixResourceManager.getHelixInstanceConfig(server); InstanceConfig realInstanceConfig = _helixAdmin.getInstanceConfig(_helixClusterName, server); Assert.assertEquals(cachedInstanceConfig, realInstanceConfig); } ZkClient zkClient = new ZkClient(_helixResourceManager.getHelixZkURL(), CONNECTION_TIMEOUT_IN_MILLISECOND, CONNECTION_TIMEOUT_IN_MILLISECOND, new ZNRecordSerializer()); modifyExistingInstanceConfig(zkClient); addAndRemoveNewInstanceConfig(zkClient); zkClient.close(); }
InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(_clusterName, instance); instanceConfig.getRecord().setBooleanField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, true); _helixAdmin.setInstanceConfig(_clusterName, instance, instanceConfig); InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(_clusterName, instance); instanceConfig.getRecord().setBooleanField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, false); _helixAdmin.setInstanceConfig(_clusterName, instance, instanceConfig); InstanceConfig instanceConfig = _helixAdmin.getInstanceConfig(_clusterName, instanceName); instanceConfig.getRecord().setBooleanField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, true); _helixAdmin.setInstanceConfig(_clusterName, instanceName, instanceConfig);
@DELETE @Path("{instanceName}") public Response deleteInstance(@PathParam("clusterId") String clusterId, @PathParam("instanceName") String instanceName) { HelixAdmin admin = getHelixAdmin(); try { InstanceConfig instanceConfig = admin.getInstanceConfig(clusterId, instanceName); admin.dropInstance(clusterId, instanceConfig); } catch (HelixException e) { return badRequest(e.getMessage()); } return OK(); }
/** * Helix participant cannot pre-configure tags before it connects to ZK. So this method can only be invoked after * {@link HelixManager#connect()}. However this will still work because tagged jobs won't be sent to a non-tagged instance. Hence * the job with EXAMPLE_INSTANCE_TAG will remain in the ZK until an instance with EXAMPLE_INSTANCE_TAG was found. */ private void addInstanceTags() { if (this.helixManager.isConnected()) { List<String> tags = ConfigUtils.getStringList(this.config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY); logger.info("Adding tags binding " + tags); tags.forEach(tag -> helixManager.getClusterManagmentTool().addInstanceTag(this.clusterName, this.helixInstanceName, tag)); logger.info("Actual tags binding " + helixManager.getClusterManagmentTool() .getInstanceConfig(this.clusterName, this.helixInstanceName).getTags()); } }
public FileStoreStateModel(HelixManager manager, String resource, String partition) { String clusterName = manager.getClusterName(); String instanceName = manager.getInstanceName(); instanceConfig = manager.getClusterManagmentTool().getInstanceConfig(clusterName, instanceName); replicator = new Replicator(instanceConfig, resource, partition); try { manager.addExternalViewChangeListener(replicator); } catch (Exception e) { e.printStackTrace(); } _partition = partition; _serverId = instanceName; }
/** * Validate instances for each partition is on different zone and with necessary tagged instances. */ private void validateZoneAndTagIsolation(IdealState is, ExternalView ev, int expectedReplica) { String tag = is.getInstanceGroupTag(); for (String partition : is.getPartitionSet()) { Set<String> assignedZones = new HashSet<String>(); Map<String, String> assignmentMap = ev.getRecord().getMapField(partition); Set<String> instancesInEV = assignmentMap.keySet(); // TODO: preference List is not persisted in IS. //Assert.assertEquals(instancesInEV, instancesInIs); for (String instance : instancesInEV) { assignedZones.add(_nodeToZoneMap.get(instance)); if (tag != null) { InstanceConfig config = _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instance); Assert.assertTrue(config.containsTag(tag)); } } Assert.assertEquals(assignedZones.size(), expectedReplica); } }
String instanceName = cmd.getOptionValues(listInstanceInfo)[1]; InstanceConfig config = setupTool.getClusterManagementTool().getInstanceConfig(clusterName, instanceName);
String instanceName = cmd.getOptionValues(listInstanceInfo)[1]; InstanceConfig config = setupTool.getClusterManagementTool().getInstanceConfig(clusterName, instanceName);
/** * Validate each partition is different instances and with necessary tagged instances. */ private void validateIsolation(IdealState is, ExternalView ev, int expectedReplica) { String tag = is.getInstanceGroupTag(); for (String partition : is.getPartitionSet()) { Map<String, String> assignmentMap = ev.getRecord().getMapField(partition); Set<String> instancesInEV = assignmentMap.keySet(); Assert.assertEquals(instancesInEV.size(), expectedReplica); for (String instance : instancesInEV) { if (tag != null) { InstanceConfig config = _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instance); Assert.assertTrue(config.containsTag(tag)); } } } }
private void setParticipantsCapacity(int perNodeTaskLimitation) { for (int i = 0; i < _numNodes; i++) { InstanceConfig instanceConfig = _gSetupTool.getClusterManagementTool() .getInstanceConfig(CLUSTER_NAME, PARTICIPANT_PREFIX + "_" + (_startPort + i)); instanceConfig.setMaxConcurrentTask(perNodeTaskLimitation); _gSetupTool.getClusterManagementTool() .setInstanceConfig(CLUSTER_NAME, PARTICIPANT_PREFIX + "_" + (_startPort + i), instanceConfig); } } }
@Test public void testEnableDisablePartitions() { String className = TestHelper.getTestClassName(); String methodName = TestHelper.getTestMethodName(); String clusterName = className + "_" + methodName; String instanceName = "TestInstance"; String testResourcePrefix = "TestResource"; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); HelixAdmin admin = new ZKHelixAdmin(_gZkClient); admin.addCluster(clusterName, true); admin.addInstance(clusterName, new InstanceConfig(instanceName)); // Test disable instances with resources admin.enablePartition(false, clusterName, instanceName, testResourcePrefix + "0", Arrays.asList(new String[]{"1", "2"})); admin.enablePartition(false, clusterName, instanceName, testResourcePrefix + "1", Arrays.asList(new String[]{"2", "3", "4"})); InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, instanceName); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "0").size(), 2); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "1").size(), 3); // Test disable partition across resources // TODO : Remove this part once setInstanceEnabledForPartition(partition, enabled) is removed instanceConfig.setInstanceEnabledForPartition("10", false); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "0").size(), 3); Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix + "1").size(), 4); admin.dropCluster(clusterName); }
liveInstanceMap.put(instanceName, liveInstance); instanceConfigMap.put(instanceName, _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, instanceName));
oldParticipant.syncStop(); InstanceConfig instanceConfig = _gSetupTool.getClusterManagementTool().getInstanceConfig(CLUSTER_NAME, oldParticipantName);
config = tool.getInstanceConfig(clusterName, instanceName); AssertJUnit.assertEquals(config.getId(), instanceName); config = tool.getInstanceConfig(clusterName, instanceName); config.setHostName("host2"); try { config = tool.getInstanceConfig(clusterName, instanceName); config.setPort("7777"); try { config = tool.getInstanceConfig(clusterName, instanceName); config.getRecord().setListField("dummy", dummyList); AssertJUnit.assertTrue(tool.setInstanceConfig(clusterName, "host1_9999", config)); config = tool.getInstanceConfig(clusterName, "host1_9999"); dummyList = config.getRecord().getListField("dummy"); AssertJUnit.assertTrue(dummyList.contains("foo")); tool.getInstanceConfig(clusterName, "host1_9999"); Assert.fail("should fail if get a non-existent instance"); } catch (HelixException e) {