/** * initializes resource usages on node */ private void intializeResources() { for (Entry<String, Map<String, Collection<ExecutorDetails>>> entry : _topIdToUsedSlots.entrySet()) { String topoId = entry.getKey(); Map<String, Collection<ExecutorDetails>> assignment = entry.getValue(); Map<ExecutorDetails, Double> topoMemoryResourceList = _topologies.getById(topoId).getTotalMemoryResourceList(); for (Collection<ExecutorDetails> execs : assignment.values()) { for (ExecutorDetails exec : execs) { if (!_isAlive) { continue; // We do not free the assigned slots (the orphaned slots) on the inactive supervisors // The inactive node will be treated as a 0-resource node and not available for other unassigned workers } if (topoMemoryResourceList.containsKey(exec)) { consumeResourcesforTask(exec, _topologies.getById(topoId)); } else { throw new IllegalStateException("Executor " + exec + "not found!"); } } } } }
/** * Schedule executor exec from topology td * * @param exec the executor to schedule * @param td the topology executor exec is a part of * @param schedulerAssignmentMap the assignments already calculated * @param scheduledTasks executors that have been scheduled */ private void scheduleExecutor(ExecutorDetails exec, TopologyDetails td, Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap, Collection<ExecutorDetails> scheduledTasks) { WorkerSlot targetSlot = this.findWorkerForExec(exec, td, schedulerAssignmentMap); if (targetSlot != null) { RAS_Node targetNode = this.idToNode(targetSlot.getNodeId()); if (!schedulerAssignmentMap.containsKey(targetSlot)) { schedulerAssignmentMap.put(targetSlot, new LinkedList<ExecutorDetails>()); } schedulerAssignmentMap.get(targetSlot).add(exec); targetNode.consumeResourcesforTask(exec, td); scheduledTasks.add(exec); LOG.debug("TASK {} assigned to Node: {} avail [ mem: {} cpu: {} ] total [ mem: {} cpu: {} ] on slot: {} on Rack: {}", exec, targetNode.getHostname(), targetNode.getAvailableMemoryResources(), targetNode.getAvailableCpuResources(), targetNode.getTotalMemoryResources(), targetNode.getTotalCpuResources(), targetSlot, nodeToRack(targetNode)); } else { LOG.error("Not Enough Resources to schedule Task {}", exec); } }
targetNode.consumeResourcesforTask(exec, td);