Double supervisorUsedMemory = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedMemory += topology.getTotalMemReqTask(executor);
for (ExecutorDetails exec : execs) { cpuUsed += topo.getTotalCpuReqTask(exec); memoryUsed += topo.getTotalMemReqTask(exec);
List<Double> assignedExecutorCpu = new ArrayList<>(); for (ExecutorDetails executor : executors1) { assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor)); assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor)); for (ExecutorDetails executor : entry.getValue()) { supervisorUsedMemory += topology1.getTotalCpuReqTask(executor); supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
/** * Get the amount of memory already assigned to a worker * * @param ws the worker to get the amount of memory assigned to a worker * @param td the topology that has executors running on the worker * @param scheduleAssignmentMap the schedulings calculated so far * @return the amount of memory */ private Double getWorkerScheduledMemoryUse(WorkerSlot ws, TopologyDetails td, Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) { Double totalMem = 0.0; Collection<ExecutorDetails> execs = scheduleAssignmentMap.get(ws); if (execs != null) { for (ExecutorDetails exec : execs) { totalMem += td.getTotalMemReqTask(exec); } } return totalMem; }
/** * Gets the total memory resource list for a * set of tasks that is part of a topology. * @return Map<ExecutorDetails, Double> a map of the total memory requirement * for all tasks in topology topoId. */ public Map<ExecutorDetails, Double> getTotalMemoryResourceList() { Map<ExecutorDetails, Double> ret = new HashMap<>(); for (ExecutorDetails exec : this.resourceList.keySet()) { ret.put(exec, getTotalMemReqTask(exec)); } return ret; }
/** * Checks whether we can schedule an Executor exec on the worker slot ws * Only considers memory currently. May include CPU in the future * * @param exec the executor to check whether it can be asssigned to worker ws * @param ws the worker to check whether executor exec can be assigned to it * @param td the topology that the exec is from * @param scheduleAssignmentMap the schedulings calculated so far * @return a boolean: True denoting the exec can be scheduled on ws and false if it cannot */ private boolean checkWorkerConstraints(ExecutorDetails exec, WorkerSlot ws, TopologyDetails td, Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) { boolean retVal = false; if (this.getWorkerScheduledMemoryAvailable(ws, td, scheduleAssignmentMap) >= td.getTotalMemReqTask(exec)) { retVal = true; } return retVal; }
/** * get the amount of memory used by a worker */ public double getMemoryUsedByWorker(WorkerSlot ws) { TopologyDetails topo = findTopologyUsingWorker(ws); if (topo == null) { return 0.0; } Collection<ExecutorDetails> execs = getExecutors(ws, _cluster); double totalMemoryUsed = 0.0; for (ExecutorDetails exec : execs) { totalMemoryUsed += topo.getTotalMemReqTask(exec); } return totalMemoryUsed; }
/** * frees the amount of resources for a executor in a topology. * @param exec is the executor for which the resources are freed for * @param topo the topology the executor is a part */ public void freeResourcesForTask(ExecutorDetails exec, TopologyDetails topo) { Double taskMemReq = topo.getTotalMemReqTask(exec); Double taskCpuReq = topo.getTotalCpuReqTask(exec); freeCPU(taskCpuReq); freeMemory(taskMemReq); } }
/** * Consumes a certain amount of resources for a executor in a topology. * @param exec is the executor that is consuming resources on this node * @param topo the topology the executor is a part */ public void consumeResourcesforTask(ExecutorDetails exec, TopologyDetails topo) { Double taskMemReq = topo.getTotalMemReqTask(exec); Double taskCpuReq = topo.getTotalCpuReqTask(exec); consumeCPU(taskCpuReq); consumeMemory(taskMemReq); }
/** * Get the best worker to assign executor exec on a rack * * @param exec the executor to schedule * @param td the topology that the executor is a part of * @param rackId the rack id of the rack to find a worker on * @param scheduleAssignmentMap already calculated assignments * @return a worker to assign executor exec to. Returns null if a worker cannot be successfully found on rack with rackId */ private WorkerSlot getBestWorker(ExecutorDetails exec, TopologyDetails td, String rackId, Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) { if (!_rackIdToSortedNodes.containsKey(rackId)) { _rackIdToSortedNodes.put(rackId, sortNodes(this.getAvailableNodesFromRack(rackId), rackId, td.getId(), scheduleAssignmentMap)); } TreeSet<ObjectResources> sortedNodes = _rackIdToSortedNodes.get(rackId); double taskMem = td.getTotalMemReqTask(exec); double taskCPU = td.getTotalCpuReqTask(exec); for (ObjectResources nodeResources : sortedNodes) { RAS_Node n = _nodes.getNodeById(nodeResources.id); if (n.getAvailableCpuResources() >= taskCPU && n.getAvailableMemoryResources() >= taskMem && n.getFreeSlots().size() > 0) { for (WorkerSlot ws : n.getFreeSlots()) { if (checkWorkerConstraints(exec, ws, td, scheduleAssignmentMap)) { return ws; } } } } return null; }