public synchronized void replaceWorkerInMirrorMaker(Map<String, List<String>> pipelineToRouteIdToReplace, List<String> workerToReplace) { _lock.lock(); try { LOGGER.info("replace: {}", _availableWorkerList); for (String pipeline : pipelineToRouteIdToReplace.keySet()) { LOGGER.info("replace: {} : {}", pipeline, pipelineToRouteIdToReplace.get(pipeline)); // TODO: if there aren't enough workers _helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.resetCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, workerToReplace, _availableWorkerList, _conf.getMaxNumWorkersPerRoute())); } } finally { _lock.unlock(); } }
public synchronized void removeWorkersToMirrorMaker(InstanceTopicPartitionHolder controller, String pipeline, int routeId, int numWorkersToRemove) throws Exception { LOGGER.info("Trying to remove {} workers in route: {}@{}", numWorkersToRemove, pipeline, routeId); _lock.lock(); try { List<String> instancesToRemove = new ArrayList<>(); Iterator<String> currWorkerIter = controller.getWorkerSet().iterator(); int count = 0; while (count < numWorkersToRemove && currWorkerIter.hasNext()) { instancesToRemove.add(currWorkerIter.next()); count++; } LOGGER.info("Remove {} instance int route {}: {}", instancesToRemove.size(), pipeline + SEPARATOR + routeId, instancesToRemove); _helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.shrinkInstanceCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, String.valueOf(routeId), instancesToRemove, _conf.getMaxNumWorkersPerRoute())); TopicPartition route = new TopicPartition(pipeline, routeId); _routeToInstanceMap.putIfAbsent(route, new ArrayList<>()); _routeToInstanceMap.get(route).removeAll(instancesToRemove); _availableWorkerList.addAll(instancesToRemove); controller.removeWorkers(instancesToRemove); } finally { _lock.unlock(); } }
_helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.expandCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, String.valueOf(routeId), instances, _conf.getMaxNumWorkersPerRoute()));
_helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.expandCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, String.valueOf(routeId), instances, _conf.getMaxNumWorkersPerRoute())); _helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.expandInstanceCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, String.valueOf(routeId), instances, _conf.getMaxNumWorkersPerRoute())); } else { LOGGER.info("Topic {} Partition {} does not exist", pipeline, routeId); _helixAdmin.setResourceIdealState(_helixClusterName, pipeline, IdealStateBuilder.expandCustomIdealStateFor(_helixAdmin.getResourceIdealState(_helixClusterName, pipeline), pipeline, String.valueOf(routeId), instances, _conf.getMaxNumWorkersPerRoute()));
_initMaxWorkloadPerWorkerByteXdc = managerConf.getInitMaxWorkloadPerWorkerByteXdc(); _initMaxNumWorkersPerRoute = managerConf.getInitMaxNumWorkersPerRoute(); _maxNumWorkersPerRoute = managerConf.getMaxNumWorkersPerRoute(); _workloadRefreshPeriodInSeconds = managerConf.getWorkloadRefreshPeriodInSeconds(); _workerHelixManager = new WorkerHelixManager(managerConf);
Assert.assertEquals(conf.getMaxNumPartitionsPerRoute().toString(), "20"); Assert.assertEquals(conf.getInitMaxNumWorkersPerRoute().toString(), "3"); Assert.assertEquals(conf.getMaxNumWorkersPerRoute().toString(), "5"); } catch (Exception e) { Assert.fail("No exception is expected");
Assert.assertEquals(conf.getMaxNumPartitionsPerRoute().toString(), "30"); Assert.assertEquals(conf.getInitMaxNumWorkersPerRoute().toString(), "20"); Assert.assertEquals(conf.getMaxNumWorkersPerRoute().toString(), "30"); Assert.assertEquals(conf.getProperty("kafka.cluster.zkStr.cluster1"), "localhost:12026/cluster1"); Assert.assertEquals(conf.getProperty("kafka.cluster.servers.cluster1"), "localhost:9091");