@Override public boolean isBlackListed(String supervisorId) { return blackListedHosts.contains(getHost(supervisorId)); }
protected Map<String, Set<String>> createHostToSupervisorMap(final List<String> blacklistedNodeIds, Cluster cluster) { Map<String, Set<String>> hostToSupervisorMap = new TreeMap<>(); for (String supervisorId : blacklistedNodeIds) { String hostname = cluster.getHost(supervisorId); if (hostname != null) { Set<String> supervisorIds = hostToSupervisorMap.get(hostname); if (supervisorIds == null) { supervisorIds = new HashSet<>(); hostToSupervisorMap.put(hostname, supervisorIds); } supervisorIds.add(supervisorId); } } return hostToSupervisorMap; } }
private Set<String> getBlacklistHosts(Cluster cluster, Topologies topologies) { Set<String> blacklistSet = blacklistStrategy.getBlacklist(new ArrayList<>(badSupervisorsToleranceSlidingWindow), cluster, topologies); Set<String> blacklistHostSet = new HashSet<>(); for (String supervisor : blacklistSet) { String host = cluster.getHost(supervisor); if (host != null) { blacklistHostSet.add(host); } else { LOG.info("supervisor {} is not alive, do not need to add to blacklist.", supervisor); } } return blacklistHostSet; }
private Map<String, Set<WorkerSlot>> hostToUsedSlots(Cluster cluster) { Collection<WorkerSlot> usedSlots = cluster.getUsedSlots(); Map<String, Set<WorkerSlot>> hostUsedSlots = new HashMap<String, Set<WorkerSlot>>(); for (WorkerSlot slot : usedSlots) { String host = cluster.getHost(slot.getNodeId()); Set<WorkerSlot> slots = hostUsedSlots.get(host); if (slots == null) { slots = new HashSet<WorkerSlot>(); hostUsedSlots.put(host, slots); } slots.add(slot); } return hostUsedSlots; }
private LinkedList<HostAssignableSlots> hostAssignableSlots(Cluster cluster) { List<WorkerSlot> assignableSlots = cluster.getAssignableSlots(); Map<String, List<WorkerSlot>> hostAssignableSlots = new HashMap<String, List<WorkerSlot>>(); for (WorkerSlot slot : assignableSlots) { String host = cluster.getHost(slot.getNodeId()); List<WorkerSlot> slots = hostAssignableSlots.get(host); if (slots == null) { slots = new ArrayList<WorkerSlot>(); hostAssignableSlots.put(host, slots); } slots.add(slot); } List<HostAssignableSlots> sortHostAssignSlots = new ArrayList<HostAssignableSlots>(); for (Map.Entry<String, List<WorkerSlot>> entry : hostAssignableSlots.entrySet()) { sortHostAssignSlots.add(new HostAssignableSlots(entry.getKey(), entry.getValue())); } Collections.sort(sortHostAssignSlots, new Comparator<HostAssignableSlots>() { @Override public int compare(HostAssignableSlots o1, HostAssignableSlots o2) { return o2.getWorkerSlots().size() - o1.getWorkerSlots().size(); } }); Collections.shuffle(sortHostAssignSlots); return new LinkedList<HostAssignableSlots>(sortHostAssignSlots); }
private Map<String, List<AssignmentInfo>> hostAssignments(Cluster cluster) { Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values(); Map<String, List<AssignmentInfo>> hostAssignments = new HashMap<String, List<AssignmentInfo>>(); for (SchedulerAssignment sa : assignmentValues) { Map<WorkerSlot, List<ExecutorDetails>> slotExecutors = Utils.reverseMap(sa.getExecutorToSlot()); Set<Map.Entry<WorkerSlot, List<ExecutorDetails>>> entries = slotExecutors.entrySet(); for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : entries) { WorkerSlot slot = entry.getKey(); List<ExecutorDetails> executors = entry.getValue(); String host = cluster.getHost(slot.getNodeId()); AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), new HashSet<ExecutorDetails>(executors)); List<AssignmentInfo> executorList = hostAssignments.get(host); if (executorList == null) { executorList = new ArrayList<AssignmentInfo>(); hostAssignments.put(host, executorList); } executorList.add(ass); } } return hostAssignments; }
public boolean isBlackListed(String supervisorId) { return blackListedHosts != null && blackListedHosts.contains(getHost(supervisorId)); }