/** * Assign a free slot on the node to the following topology and executors. * This will update the cluster too. * @param topId the topology to assign a free slot to. * @param executors the executors to run in that slot. * @param cluster the cluster to be updated */ public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) { if (!_isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + _nodeId); } if (_freeSlots.isEmpty()) { throw new IllegalStateException("Trying to assign to a full node " + _nodeId); } if (executors.size() == 0) { LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)"); } else { WorkerSlot slot = _freeSlots.iterator().next(); cluster.assign(slot, topId, executors); assignInternal(slot, topId, false); } }
public static void scheduleTopologiesEvenly(Topologies topologies, Cluster cluster) { for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { String topologyId = topology.getId(); Map<ExecutorDetails, WorkerSlot> newAssignment = scheduleTopology(topology, cluster); Map<WorkerSlot, List<ExecutorDetails>> nodePortToExecutors = Utils.reverseMap(newAssignment); for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : nodePortToExecutors.entrySet()) { WorkerSlot nodePort = entry.getKey(); List<ExecutorDetails> executors = entry.getValue(); cluster.assign(nodePort, topologyId, executors); } } }
/** * Set assignments for cluster. */ public void setAssignments( Map<String, ? extends SchedulerAssignment> newAssignments, boolean ignoreSingleExceptions) { if (newAssignments == assignments) { //NOOP return; } for (SchedulerAssignment assignment : newAssignments.values()) { assertValidTopologyForModification(assignment.getTopologyId()); } for (SchedulerAssignment assignment : assignments.values()) { assertValidTopologyForModification(assignment.getTopologyId()); } assignments.clear(); totalResourcesPerNodeCache.clear(); nodeToScheduledResourcesCache.values().forEach(Map::clear); nodeToUsedSlotsCache.values().forEach(Set::clear); for (SchedulerAssignment assignment : newAssignments.values()) { assign(assignment, ignoreSingleExceptions); } }
/** * Assign everything for the given topology. * * @param assignment the new assignment to make */ public void assign(SchedulerAssignment assignment, boolean ignoreSingleExceptions) { String id = assignment.getTopologyId(); assertValidTopologyForModification(id); Map<WorkerSlot, Collection<ExecutorDetails>> slotToExecs = assignment.getSlotToExecutors(); for (Entry<WorkerSlot, Collection<ExecutorDetails>> entry : slotToExecs.entrySet()) { try { assign(entry.getKey(), id, entry.getValue()); } catch (RuntimeException e) { if (!ignoreSingleExceptions) { throw e; } } } }
cluster.assign(target, td.getId(), executors);
for (WorkerSlot tmpSlot : slot.subList(0, num)) { Set<ExecutorDetails> executor = removeElemFromExecutorsSet(executorSet); cluster.assign(tmpSlot, topologyId, executor);
@Test public void testResourcePoolUtilization() { INimbus iNimbus = new INimbusTest(); Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000); Double cpuGuarantee = 400.0; Double memoryGuarantee = 1000.0; Map<String, Map<String, Number>> resourceUserPool = userResourcePool( userRes("user1", cpuGuarantee, memoryGuarantee)); Config config = createClusterConfig(100, 200, 200, resourceUserPool); TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9, "user1"); Topologies topologies = new Topologies(topo1); Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config); User user1 = new User("user1", toDouble(resourceUserPool.get("user1"))); WorkerSlot slot = cluster.getAvailableSlots().get(0); cluster.assign(slot, topo1.getId(), topo1.getExecutors()); Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001); Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001); Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001); Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001); } }
/** * Assign a free slot on the node to the following topology and executors. * This will update the cluster too. * @param topId the topology to assign a free slot to. * @param executors the executors to run in that slot. * @param cluster the cluster to be updated */ public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) { if (!_isAlive) { throw new IllegalStateException("Trying to adding to a dead node " + _nodeId); } if (_freeSlots.isEmpty()) { throw new IllegalStateException("Trying to assign to a full node " + _nodeId); } if (executors.size() == 0) { LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)"); } else { WorkerSlot slot = _freeSlots.iterator().next(); cluster.assign(slot, topId, executors); assignInternal(slot, topId, false); } }
_cluster.assign(target, td.getId(), executors);
slotAssignmentStrings.add("(" + mesosWorkerSlots.get(0).getNodeId() + ":" + mesosWorkerSlots.get(0).getPort() + ", " + executorsPerWorker.toString() + ")"); iterator.remove(); cluster.assign(mesosWorkerSlots.remove(0), topologyId, executorsPerWorker);