private static IScheduler makeScheduler(Map<String, Object> conf, INimbus inimbus) { String schedClass = (String) conf.get(DaemonConfig.STORM_SCHEDULER); IScheduler scheduler = inimbus == null ? null : inimbus.getForcedScheduler(); if (scheduler != null) { LOG.info("Using forced scheduler from INimbus {} {}", scheduler.getClass(), scheduler); } else if (schedClass != null) { LOG.info("Using custom scheduler: {}", schedClass); scheduler = ReflectionUtils.newInstance(schedClass); } else { LOG.info("Using default scheduler"); scheduler = new DefaultScheduler(); } return scheduler; }
public static void defaultSchedule(Topologies topologies, Cluster cluster) { for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { List<WorkerSlot> availableSlots = cluster.getAvailableSlots(); Set<ExecutorDetails> allExecutors = topology.getExecutors(); Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = EvenScheduler.getAliveAssignedWorkerSlotExecutors(cluster, topology.getId()); Set<ExecutorDetails> aliveExecutors = new HashSet<ExecutorDetails>(); for (List<ExecutorDetails> list : aliveAssigned.values()) { aliveExecutors.addAll(list); } Set<WorkerSlot> canReassignSlots = slotsCanReassign(cluster, aliveAssigned.keySet()); int totalSlotsToUse = Math.min(topology.getNumWorkers(), canReassignSlots.size() + availableSlots.size()); Set<WorkerSlot> badSlots = null; if (totalSlotsToUse > aliveAssigned.size() || !allExecutors.equals(aliveExecutors)) { badSlots = badSlots(aliveAssigned, allExecutors.size(), totalSlotsToUse); } if (badSlots != null) { cluster.freeSlots(badSlots); } EvenScheduler.scheduleTopologiesEvenly(new Topologies(topology), cluster); } }
@Override public void schedule(Topologies topologies, Cluster cluster) { defaultSchedule(topologies, cluster); }
topoMap.put(topo2.getId(), topo2); Topologies topologies = new Topologies(topoMap); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), new StormMetricsRegistry()); bs.prepare(config);
DefaultScheduler.defaultSchedule(leftOverTopologies, cluster);
ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies,cluster);
@Test public void TestBadSlot() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster);
@Test public void TestBadSupervisor() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster);