WorkerSlot workerSlot = new WorkerSlot(supervisor, slot); int slotCount = slotCountMap.getOrDefault(workerSlot, 0); slotCountMap.put(workerSlot, slotCount + 1); String supervisorKey = workerSlot.getNodeId(); Integer slot = workerSlot.getPort(); int value = entry.getValue(); if (value == windowSize) { // worker slot which was never back to normal in tolerance period will be removed from cache
assignment.getSlotToExecutors().entrySet()) { WorkerSlot slot = entry.getKey(); String nodeId = slot.getNodeId(); Collection<ExecutorDetails> execs = entry.getValue(); if (!assignmentRelationshipMap.containsKey(nodeId)) { workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>()); workerIdToWorker.get(nodeId).put(slot.getId(), slot); if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) { assignmentRelationshipMap .put(topId, new HashMap<String, Collection<ExecutorDetails>>()); if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) { assignmentRelationshipMap .get(nodeId) .get(topId) .put(slot.getId(), new LinkedList<ExecutorDetails>()); assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs); WorkerSlot worker = new WorkerSlot(sup.getId(), port); if (!workerIdToWorker.containsKey(sup.getId())) { workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>()); if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) { workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
private void validateSlot(WorkerSlot ws) { if (!_nodeId.equals(ws.getNodeId())) { throw new IllegalArgumentException( "Trying to add a slot to the wrong node " + ws + " is not a part of " + _nodeId); } }
public Node(String nodeId, Set<Integer> allPorts, boolean isAlive) { _nodeId = nodeId; _isAlive = isAlive; if (_isAlive && allPorts != null) { for (int port : allPorts) { _freeSlots.add(new WorkerSlot(_nodeId, port)); } } }
WorkerSlot targetSlot = workerToTasksEntry.getKey(); Collection<ExecutorDetails> execsNeedScheduling = workerToTasksEntry.getValue(); RAS_Node targetNode = this.schedulingState.nodes.getNodeById(targetSlot.getNodeId()); td.getName(), execsNeedScheduling, targetNode.getHostname(), targetSlot.getPort()); nodesUsed.add(targetNode.getId()); assignedMemOnHeap += targetSlot.getAllocatedMemOnHeap(); assignedMemOffHeap += targetSlot.getAllocatedMemOffHeap(); assignedCpu += targetSlot.getAllocatedCpu(); targetSlot.getAllocatedMemOnHeap(), targetSlot.getAllocatedMemOffHeap(), targetSlot.getAllocatedCpu()}; workerResources.put (targetSlot, worker_resources);
@Override public String toString() { return getId(); } }
List<ExecutorDetails> failedExecutors = new ArrayList<>(); for (Map.Entry<ExecutorDetails, WorkerSlot> entry : executorToSlot.entrySet()) { if (entry.getValue().equals(failedWorker)) { failedExecutors.add(entry.getKey()); executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", 0)); executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", 1)); executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1)); Map<String, SchedulerAssignment> existingAssignments = new HashMap<>(); assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null); executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", 1)); // the worker to orphan executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", 2)); // the worker that fails executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1)); // the healthy worker existingAssignments = new HashMap<>(); assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null);
/** * Release the slot occupied by this assignment. */ public void unassignBySlot(WorkerSlot slot) { List<ExecutorDetails> executors = new ArrayList<>(); for (ExecutorDetails executor : this.executorToSlot.keySet()) { WorkerSlot ws = this.executorToSlot.get(executor); if (ws.equals(slot)) { executors.add(executor); } } // remove for (ExecutorDetails executor : executors) { this.executorToSlot.remove(executor); } }
@Override public boolean isSlotOccupied(WorkerSlot slot) { return nodeToUsedSlotsCache.computeIfAbsent(slot.getNodeId(), MAKE_SET).contains(slot); }
@Override public List<WorkerSlot> getAssignableSlots(SupervisorDetails supervisor) { Set<Integer> ports = this.getAssignablePorts(supervisor); List<WorkerSlot> slots = new ArrayList<>(ports.size()); for (Integer port : ports) { slots.add(new WorkerSlot(supervisor.getId(), port)); } return slots; }
throw new IllegalStateException("Trying to assign already used slot" + target.getPort() + "on node " + _nodeId); if (!_topIdToUsedSlots.get(td.getId()).containsKey(target.getId())) { _topIdToUsedSlots.get(td.getId()).put(target.getId(), new LinkedList<ExecutorDetails>()); _topIdToUsedSlots.get(td.getId()).get(target.getId()).addAll(executors);
/** * Find a which topology is running on a worker slot. * * @return the topology using the worker slot. If worker slot is free then return null */ private TopologyDetails findTopologyUsingWorker(WorkerSlot ws) { for (Entry<String, Map<String, Collection<ExecutorDetails>>> entry : topIdToUsedSlots.entrySet()) { String topoId = entry.getKey(); Set<String> workerIds = entry.getValue().keySet(); for (String workerId : workerIds) { if (ws.getId().equals(workerId)) { return cluster.getTopologies().getById(topoId); } } } return null; }
public static Set<WorkerSlot> slotsCanReassign(Cluster cluster, Set<WorkerSlot> slots) { Set<WorkerSlot> result = new HashSet<WorkerSlot>(); for (WorkerSlot slot : slots) { if (!cluster.isBlackListed(slot.getNodeId())) { SupervisorDetails supervisor = cluster.getSupervisorById(slot.getNodeId()); if (supervisor != null) { Set<Integer> ports = supervisor.getAllPorts(); if (ports != null && ports.contains(slot.getPort())) { result.add(slot); } } } } return result; }
/** * Release the slot occupied by this assignment. */ public void unassignBySlot(WorkerSlot slot) { Collection<ExecutorDetails> executors = slotToExecutors.remove(slot); // remove if (executors != null) { for (ExecutorDetails executor : executors) { executorToSlot.remove(executor); } } resources.remove(slot); String node = slot.getNodeId(); boolean isFound = false; for (WorkerSlot ws : executorToSlot.values()) { if (node.equals(ws.getNodeId())) { isFound = true; break; } } if (!isFound) { nodeIdToTotalSharedOffHeap.remove(node); } }
private WorkerSlot allocateResourceToSlot (TopologyDetails td, Collection<ExecutorDetails> executors, WorkerSlot slot) { double onHeapMem = 0.0; double offHeapMem = 0.0; double cpu = 0.0; for (ExecutorDetails exec : executors) { Double onHeapMemForExec = td.getOnHeapMemoryRequirement(exec); if (onHeapMemForExec != null) { onHeapMem += onHeapMemForExec; } Double offHeapMemForExec = td.getOffHeapMemoryRequirement(exec); if (offHeapMemForExec != null) { offHeapMem += offHeapMemForExec; } Double cpuForExec = td.getTotalCpuReqTask(exec); if (cpuForExec != null) { cpu += cpuForExec; } } return new WorkerSlot(slot.getNodeId(), slot.getPort(), onHeapMem, offHeapMem, cpu); }
String nodeId = slot.getNodeId(); Collection<ExecutorDetails> execs = entry.getValue(); if (!assignmentRelationshipMap.containsKey(nodeId)) { workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>()); workerIdToWorker.get(nodeId).put(slot.getId(), slot); if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) { assignmentRelationshipMap.get(nodeId).put(topId, new HashMap<String, Collection<ExecutorDetails>>()); if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) { assignmentRelationshipMap.get(nodeId).get(topId).put(slot.getId(), new LinkedList<ExecutorDetails>()); assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs); WorkerSlot worker = new WorkerSlot(sup.getId(), port); if (!workerIdToWorker.containsKey(sup.getId())) { workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>()); if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) { workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
@Override public List<WorkerSlot> getAvailableSlots(SupervisorDetails supervisor) { Set<Integer> ports = this.getAvailablePorts(supervisor); List<WorkerSlot> slots = new ArrayList<>(ports.size()); for (Integer port : ports) { slots.add(new WorkerSlot(supervisor.getId(), port)); } return slots; }