/** * Update the assignments and status from the other cluster. * * @param other the cluster to get the assignments and status from */ public void updateFrom(Cluster other) { for (SchedulerAssignment assignment : other.getAssignments().values()) { assertValidTopologyForModification(assignment.getTopologyId()); } setAssignments(other.getAssignments(), false); setStatusMap(other.getStatusMap()); } }
private Map<String, List<AssignmentInfo>> hostAssignments(Cluster cluster) { Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values(); Map<String, List<AssignmentInfo>> hostAssignments = new HashMap<String, List<AssignmentInfo>>(); for (SchedulerAssignment sa : assignmentValues) { Map<WorkerSlot, List<ExecutorDetails>> slotExecutors = Utils.reverseMap(sa.getExecutorToSlot()); Set<Map.Entry<WorkerSlot, List<ExecutorDetails>>> entries = slotExecutors.entrySet(); for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : entries) { WorkerSlot slot = entry.getKey(); List<ExecutorDetails> executors = entry.getValue(); String host = cluster.getHost(slot.getNodeId()); AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), new HashSet<ExecutorDetails>(executors)); List<AssignmentInfo> executorList = hostAssignments.get(host); if (executorList == null) { executorList = new ArrayList<AssignmentInfo>(); hostAssignments.put(host, executorList); } executorList.add(ass); } } return hostAssignments; }
for (SchedulerAssignment assignment : cluster.getAssignments().values()) { String topId = assignment.getTopologyId();
for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) { String topId = entry.getValue().getTopologyId(); for (WorkerSlot ws : entry.getValue().getSlots()) {
return cluster.getAssignments();
public static Cluster nextCluster(Cluster cluster, Map<String, SupervisorDetails> supervisors, INimbus iNimbus, Map<String, Object> config, Topologies topologies) { Map<String, SchedulerAssignmentImpl> assignment; if (cluster == null) { assignment = new HashMap<>(); } else { assignment = TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()); } return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, assignment, topologies, config); } }
rs.schedule(topologies, cluster); Map<String, SchedulerAssignment> assignments = new TreeMap<>(cluster.getAssignments()); assertEquals(2, assignments.size());
LOG.info("HAS {} SLOTS USED", cluster.getUsedSlots().size()); Map<String, SchedulerAssignment> assignments = new TreeMap<>(cluster.getAssignments());
bs.prepare(config); bs.schedule(topologies,cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,"sup-0"),TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); for (int i = 0 ; i < 20 ; i++){ bs.schedule(topologies,cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies,cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap,"sup-0",0),TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); for (int i = 0 ;i < 20 ; i++){ bs.schedule(topologies, cluster);
@Test public void TestBadSupervisor() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); topoMap.put(topo4.getId(), topo4); topologies = new Topologies(topoMap); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
for (SchedulerAssignment assignment : cluster.getAssignments().values()) { for (Entry<WorkerSlot, WorkerResources> entry : new HashMap<>(assignment.getScheduledResources()).entrySet()) { WorkerSlot ws = entry.getKey();
@Test public void TestBadSlot() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
/** * Get all the used ports of this supervisor. */ public Set<Integer> getUsedPorts(SupervisorDetails supervisor) { Map<String, SchedulerAssignment> assignments = this.getAssignments(); Set<Integer> usedPorts = new HashSet<Integer>(); for (SchedulerAssignment assignment : assignments.values()) { for (WorkerSlot slot : assignment.getExecutorToSlot().values()) { if (slot.getNodeId().equals(supervisor.getId())) { usedPorts.add(slot.getPort()); } } } return usedPorts; }
public static Collection<ExecutorDetails> getExecutors(WorkerSlot ws, Cluster cluster) { Collection<ExecutorDetails> retList = new ArrayList<ExecutorDetails>(); for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments() .entrySet()) { Map<ExecutorDetails, WorkerSlot> executorToSlot = entry.getValue() .getExecutorToSlot(); for (Map.Entry<ExecutorDetails, WorkerSlot> execToSlot : executorToSlot .entrySet()) { WorkerSlot slot = execToSlot.getValue(); if (ws.getPort() == slot.getPort() && ws.getNodeId().equals(slot.getNodeId())) { ExecutorDetails exec = execToSlot.getKey(); retList.add(exec); } } } return retList; }
public void updateAssignedMemoryForTopologyAndSupervisor(Topologies topologies) { Map<String, Double> supervisorToAssignedMem = new HashMap<String, Double>(); for (Map.Entry<String, SchedulerAssignment> entry : this.getAssignments().entrySet()) { String topId = entry.getValue().getTopologyId(); if (topologies.getById(topId) == null) {
for (SchedulerAssignment assignment : cluster.getAssignments().values()) { String topId = assignment.getTopologyId();
for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) { String topId = entry.getValue().getTopologyId(); for (WorkerSlot ws: entry.getValue().getSlots()) {
private void updateChanges(Cluster cluster, Topologies topologies) { //Cannot simply set this.cluster=schedulingState.cluster since clojure is immutable cluster.setAssignments(schedulingState.cluster.getAssignments()); cluster.setBlacklistedHosts(schedulingState.cluster.getBlacklistedHosts()); cluster.setStatusMap(schedulingState.cluster.getStatusMap()); cluster.setSupervisorsResourcesMap(schedulingState.cluster.getSupervisorsResourcesMap()); cluster.setTopologyResourcesMap(schedulingState.cluster.getTopologyResourcesMap()); cluster.setWorkerResourcesMap(schedulingState.cluster.getWorkerResourcesMap()); //updating resources used by supervisor updateSupervisorsResources(cluster, topologies); }