/** * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them. * * @return the total cpu requested for this topology */ public double getTotalRequestedCpu() { double totalCpu = 0.0; for (ExecutorDetails exec : this.getExecutors()) { Double execCpu = getTotalCpuReqTask(exec); if (execCpu != null) { totalCpu += execCpu; } } return totalCpu; }
Double supervisorUsedCpu = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedCpu += topology.getTotalCpuReqTask(executor);
double memoryUsed = 0.0; for (ExecutorDetails exec : execs) { cpuUsed += topo.getTotalCpuReqTask(exec); memoryUsed += topo.getTotalMemReqTask(exec);
for (ExecutorDetails executor : executors1) { assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor)); assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor)); Double supervisorUsedMemory = 0.0; for (ExecutorDetails executor : entry.getValue()) { supervisorUsedMemory += topology1.getTotalCpuReqTask(executor); supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
/** * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. * We reserve the right to change them. * * @return the total cpu requested for this topology */ public Double getTotalRequestedCpu() { Double total_cpu = 0.0; for (ExecutorDetails exec : this.getExecutors()) { Double exec_cpu = getTotalCpuReqTask(exec); if (exec_cpu != null) { total_cpu += exec_cpu; } } return total_cpu; }
/** * get the amount of cpu used by a worker */ public double getCpuUsedByWorker(WorkerSlot ws) { TopologyDetails topo = findTopologyUsingWorker(ws); if (topo == null) { return 0.0; } Collection<ExecutorDetails> execs = getExecutors(ws, _cluster); double totalCpuUsed = 0.0; for (ExecutorDetails exec : execs) { totalCpuUsed += topo.getTotalCpuReqTask(exec); } return totalCpuUsed; }
/** * Consumes a certain amount of resources for a executor in a topology. * @param exec is the executor that is consuming resources on this node * @param topo the topology the executor is a part */ public void consumeResourcesforTask(ExecutorDetails exec, TopologyDetails topo) { Double taskMemReq = topo.getTotalMemReqTask(exec); Double taskCpuReq = topo.getTotalCpuReqTask(exec); consumeCPU(taskCpuReq); consumeMemory(taskMemReq); }
/** * frees the amount of resources for a executor in a topology. * @param exec is the executor for which the resources are freed for * @param topo the topology the executor is a part */ public void freeResourcesForTask(ExecutorDetails exec, TopologyDetails topo) { Double taskMemReq = topo.getTotalMemReqTask(exec); Double taskCpuReq = topo.getTotalCpuReqTask(exec); freeCPU(taskCpuReq); freeMemory(taskMemReq); } }
/** * Get the best worker to assign executor exec on a rack * * @param exec the executor to schedule * @param td the topology that the executor is a part of * @param rackId the rack id of the rack to find a worker on * @param scheduleAssignmentMap already calculated assignments * @return a worker to assign executor exec to. Returns null if a worker cannot be successfully found on rack with rackId */ private WorkerSlot getBestWorker(ExecutorDetails exec, TopologyDetails td, String rackId, Map<WorkerSlot, Collection<ExecutorDetails>> scheduleAssignmentMap) { if (!_rackIdToSortedNodes.containsKey(rackId)) { _rackIdToSortedNodes.put(rackId, sortNodes(this.getAvailableNodesFromRack(rackId), rackId, td.getId(), scheduleAssignmentMap)); } TreeSet<ObjectResources> sortedNodes = _rackIdToSortedNodes.get(rackId); double taskMem = td.getTotalMemReqTask(exec); double taskCPU = td.getTotalCpuReqTask(exec); for (ObjectResources nodeResources : sortedNodes) { RAS_Node n = _nodes.getNodeById(nodeResources.id); if (n.getAvailableCpuResources() >= taskCPU && n.getAvailableMemoryResources() >= taskMem && n.getFreeSlots().size() > 0) { for (WorkerSlot ws : n.getFreeSlots()) { if (checkWorkerConstraints(exec, ws, td, scheduleAssignmentMap)) { return ws; } } } } return null; }
private WorkerSlot allocateResourceToSlot (TopologyDetails td, Collection<ExecutorDetails> executors, WorkerSlot slot) { double onHeapMem = 0.0; double offHeapMem = 0.0; double cpu = 0.0; for (ExecutorDetails exec : executors) { Double onHeapMemForExec = td.getOnHeapMemoryRequirement(exec); if (onHeapMemForExec != null) { onHeapMem += onHeapMemForExec; } Double offHeapMemForExec = td.getOffHeapMemoryRequirement(exec); if (offHeapMemForExec != null) { offHeapMem += offHeapMemForExec; } Double cpuForExec = td.getTotalCpuReqTask(exec); if (cpuForExec != null) { cpu += cpuForExec; } } return new WorkerSlot(slot.getNodeId(), slot.getPort(), onHeapMem, offHeapMem, cpu); }