@Override public List<WorkerSlot> getAvailableSlots() { List<WorkerSlot> slots = new ArrayList<>(); for (SupervisorDetails supervisor : this.supervisors.values()) { slots.addAll(this.getAvailableSlots(supervisor)); } return slots; }
@Override public List<WorkerSlot> getNonBlacklistedAvailableSlots(List<String> blacklistedSupervisorIds) { List<WorkerSlot> slots = new ArrayList<>(); for (SupervisorDetails supervisor : this.supervisors.values()) { if (!isBlackListed(supervisor.getId()) && !blacklistedSupervisorIds.contains(supervisor.getId())) { slots.addAll(getAvailableSlots(supervisor)); } } return slots; }
private static Map<ExecutorDetails, WorkerSlot> scheduleTopology(TopologyDetails topology, Cluster cluster) { List<WorkerSlot> availableSlots = cluster.getAvailableSlots(); Set<ExecutorDetails> allExecutors = topology.getExecutors(); Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = getAliveAssignedWorkerSlotExecutors(cluster, topology.getId());
public static void defaultSchedule(Topologies topologies, Cluster cluster) { for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { List<WorkerSlot> availableSlots = cluster.getAvailableSlots(); Set<ExecutorDetails> allExecutors = topology.getExecutors(); Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = EvenScheduler.getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); Set<ExecutorDetails> aliveExecutors = new HashSet<ExecutorDetails>(); for (List<ExecutorDetails> list : aliveAssigned.values()) { aliveExecutors.addAll(list); } Set<WorkerSlot> canReassignSlots = slotsCanReassign(cluster, aliveAssigned.keySet()); int totalSlotsToUse = Math.min(topology.getNumWorkers(), canReassignSlots.size() + availableSlots.size()); Set<WorkerSlot> badSlots = null; if (totalSlotsToUse > aliveAssigned.size() || !allExecutors.equals(aliveExecutors)) { badSlots = badSlots(aliveAssigned, allExecutors.size(), totalSlotsToUse); } if (badSlots != null) { cluster.freeSlots(badSlots); } EvenScheduler.scheduleTopologiesEvenly(new Topologies(topology), cluster); } }
@Override public void schedule(Topologies topologies, Cluster cluster) { LOG.debug("running Black List scheduler"); LOG.debug("AssignableSlots: {}", cluster.getAssignableSlots()); LOG.debug("AvailableSlots: {}", cluster.getAvailableSlots()); LOG.debug("UsedSlots: {}", cluster.getUsedSlots()); Map<String, SupervisorDetails> supervisors = cluster.getSupervisors(); blacklistStrategy.resumeFromBlacklist(); badSupervisors(supervisors); Set<String> blacklistHosts = getBlacklistHosts(cluster, topologies); this.blacklistHost = blacklistHosts; cluster.setBlacklistedHosts(blacklistHosts); removeLongTimeDisappearFromCache(); underlyingScheduler.schedule(topologies, cluster); }
@Test public void testResourcePoolUtilization() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000); Double cpuGuarantee = 400.0; Double memoryGuarantee = 1000.0; Map<String, Map<String, Number>> resourceUserPool = userResourcePool( userRes("user1", cpuGuarantee, memoryGuarantee)); Config config = createClusterConfig(100, 200, 200, resourceUserPool); TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9, "user1"); Topologies topologies = new Topologies(topo1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); User user1 = new User("user1", toDouble(resourceUserPool.get("user1"))); WorkerSlot slot = cluster.getAvailableSlots().get(0); cluster.assign(slot, topo1.getId(), topo1.getExecutors()); Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001); Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001); Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001); Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001); } }
/** * @return all the available worker slots in the cluster. */ public List<WorkerSlot> getAvailableSlots() { List<WorkerSlot> slots = new ArrayList<WorkerSlot>(); for (SupervisorDetails supervisor : this.supervisors.values()) { slots.addAll(this.getAvailableSlots(supervisor)); } return slots; }
List<WorkerSlot> workerSlots = cluster.getAvailableSlots(); String info = "";