public static Cluster nextCluster(Cluster cluster, Map<String, SupervisorDetails> supervisors, INimbus iNimbus, Map<String, Object> config, Topologies topologies) { Map<String, SchedulerAssignmentImpl> assignment; if (cluster == null) { assignment = new HashMap<>(); } else { assignment = TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()); } return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, assignment, topologies, config); } }
StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies,cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap,"sup-0"),TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); for (int i = 0 ; i < 20 ; i++){ bs.schedule(topologies,cluster); cached.add("sup-2"); Assert.assertEquals(cached,bs.cachedSupervisors.keySet()); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies,cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap,"sup-0",0),TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); for (int i = 0 ;i < 20 ; i++){ bs.schedule(topologies, cluster);
public Cluster makeCluster(TopologyDetails topo) { Topologies topologies = new Topologies(topo); Map<String, SupervisorDetails> supMap = genSupervisors(4, 2, 120, 1200); return new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, new Config()); }
@Test public void TestBadSlot() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
@Test public void TestBadSupervisor() { INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest(); Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4); Config config = new Config(); config.putAll(Utils.readDefaultConfig()); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200); config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2); config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300); Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>(); TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true); topoMap.put(topo1.getId(), topo1); Topologies topologies = new Topologies(topoMap); StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); }
@Test public void sanityTestOfScheduling() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(1, 2, 400, 2000); Config config = new Config(); config.putAll(defaultTopologyConf); ResourceAwareScheduler rs = new ResourceAwareScheduler(); TopologyDetails topology1 = genTopology("topology1", config, 1, 1, 1, 1, 0, 0, "user"); Topologies topologies = new Topologies(topology1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.prepare(config); rs.schedule(topologies, cluster); SchedulerAssignment assignment = cluster.getAssignmentById(topology1.getId()); Set<WorkerSlot> assignedSlots = assignment.getSlots(); Set<String> nodesIDs = new HashSet<>(); for (WorkerSlot slot : assignedSlots) { nodesIDs.add(slot.getNodeId()); } Collection<ExecutorDetails> executors = assignment.getExecutors(); assertEquals(1, assignedSlots.size()); assertEquals(1, nodesIDs.size()); assertEquals(2, executors.size()); assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId())); }
StormMetricsRegistry metricsRegistry = new StormMetricsRegistry(); ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry); Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler(), metricsRegistry); bs.prepare(config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts()); topoMap.put(topo4.getId(), topo4); topologies = new Topologies(topoMap); cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config); bs.schedule(topologies, cluster); Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); cluster = new Cluster(cluster, topologies); LOG.info("\n\n\t\tScheduling topos 1,3,4,5,6"); rs.schedule(topologies, cluster); cluster = new Cluster(cluster, topologies); LOG.info("\n\n\t\tScheduling topos 1-6"); rs.schedule(topologies, cluster);
genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); cluster = new Cluster(cluster, topologies); LOG.info("\n\n\t\tScheduling topos 1 to 5"); rs.schedule(topologies, cluster);
genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 29, "user"), genTopology("topo-2", config, 1, 0, 2, 0, currentTime - 2, 10, "user")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler();
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
genTopology("topo-3-bobby", config, 1, 0, 1, 0,Time.currentTimeSecs() - 300,20, "bobby"), genTopology("topo-4-derek", config, 1, 0, 1, 0,Time.currentTimeSecs() - 201,29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); genTopology("topo-5-derek", config, 1, 0, 1, 0,Time.currentTimeSecs() - 15,29, "derek")); cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster); genTopology("topo-6-bobby", config, 1, 0, 1, 0,Time.currentTimeSecs() - 10,29, "bobby")); cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); rs.schedule(topologies, cluster);
genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 15, 29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); cluster = new Cluster(cluster, topologies); rs.schedule(topologies, cluster); cluster = new Cluster(cluster, topologies); rs.schedule(topologies, cluster); cluster = new Cluster(cluster, topologies); rs.schedule(topologies, cluster);
ResourceAwareScheduler rs = new ResourceAwareScheduler(); Topologies topologies = new Topologies(topology1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1); rs.prepare(config1); rs.schedule(topologies, cluster); TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user"); topologies = new Topologies(topology2); cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config2); rs.prepare(config2); rs.schedule(topologies, cluster);
/** * When the first topology failed to be scheduled make sure subsequent schedulings can still succeed */ @Test public void testSchedulingAfterFailedScheduling() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(8, 4, 100, 1000); Config config = createClusterConfig(100, 500, 500, null); TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, "jerry"); TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry"); TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry"); Topologies topologies = new Topologies(topo1, topo2, topo3); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); rs.schedule(topologies, cluster); assertTrue("Topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null); assertEquals("Topo-2 all executors scheduled?", 4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size()); assertTrue("Topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null); assertEquals("Topo-3 all executors scheduled?", 3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size()); }
@Test public void testResourcePoolUtilization() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000); Double cpuGuarantee = 400.0; Double memoryGuarantee = 1000.0; Map<String, Map<String, Number>> resourceUserPool = userResourcePool( userRes("user1", cpuGuarantee, memoryGuarantee)); Config config = createClusterConfig(100, 200, 200, resourceUserPool); TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9, "user1"); Topologies topologies = new Topologies(topo1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); User user1 = new User("user1", toDouble(resourceUserPool.get("user1"))); WorkerSlot slot = cluster.getAvailableSlots().get(0); cluster.assign(slot, topo1.getId(), topo1.getExecutors()); Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001); Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001); Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001); Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001); } }
genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); cluster = new Cluster(cluster, topologies); rs.schedule(topologies, cluster);
@Test public void testSubmitUsersWithNoGuarantees() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000); Map<String, Map<String, Number>> resourceUserPool = userResourcePool( userRes("jerry", 200, 2000)); Config config = createClusterConfig(100, 500, 500, resourceUserPool); Topologies topologies = new Topologies( genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry"), genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby")); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler(); rs.prepare(config); rs.schedule(topologies, cluster); assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-3", "topo-4"); assertTopologiesNotScheduled(cluster, "topo-5"); }
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config); ResourceAwareScheduler rs = new ResourceAwareScheduler();