private List<TopologyDetails> isolatedTopologies(Collection<TopologyDetails> topologies) { Set<String> topologyNames = isoMachines.keySet(); List<TopologyDetails> isoTopologies = new ArrayList<TopologyDetails>(); for (TopologyDetails topo : topologies) { if (topologyNames.contains(topo.getName())) { isoTopologies.add(topo); } } return isoTopologies; }
/** * Create a new Topologies from a map of id to topology * * @param topologies a map of topology id to topology details. */ public Topologies(Map<String, TopologyDetails> topologies) { if (topologies == null) { topologies = Collections.emptyMap(); } this.topologies = new HashMap<>(topologies); this.nameToId = new HashMap<>(topologies.size()); for (Map.Entry<String, TopologyDetails> entry : topologies.entrySet()) { TopologyDetails topology = entry.getValue(); this.nameToId.put(topology.getName(), entry.getKey()); } }
private Map<Integer, Integer> machineDistribution(TopologyDetails topology) { int machineNum = isoMachines.get(topology.getName()).intValue(); int workerNum = topology.getNumWorkers(); TreeMap<Integer, Integer> distribution = Utils.integerDivided(workerNum, machineNum); if (distribution.containsKey(0)) { distribution.remove(0); } return distribution; }
/** * Get User wrappers around cluster. * * @param cluster the cluster to get the users out of. */ private Map<String, User> getUsers(Cluster cluster) { Map<String, User> userMap = new HashMap<>(); Map<String, Map<String, Double>> userResourcePools = getUserResourcePools(); LOG.debug("userResourcePools: {}", userResourcePools); for (TopologyDetails td : cluster.getTopologies()) { String topologySubmitter = td.getTopologySubmitter(); //additional safety check to make sure that topologySubmitter is going to be a valid value if (topologySubmitter == null || topologySubmitter.equals("")) { LOG.error("Cannot determine user for topology {}. Will skip scheduling this topology", td.getName()); continue; } if (!userMap.containsKey(topologySubmitter)) { userMap.put(topologySubmitter, new User(topologySubmitter, userResourcePools.get(topologySubmitter))); } } return userMap; }
LOG.error("No available slots for topology: {}", topology.getName()); return new HashMap<ExecutorDetails, WorkerSlot>();
Node n = findBestNode(allNodes); if (n == null) { LOG.error("No nodes to use to assign topology {}", td.getName()); break;
slots = ServerUtils.getEstimatedWorkerCountForRASTopo(td.getConf(), td.getTopology()); } catch (InvalidTopologyException e) { LOG.warn("Could not guess the number of slots needed for {}", td.getName(), e); needed.add(resources); LOG.warn("{} needs to be scheduled with {} and {} slots", td.getName(), resources, tdSlotsNeeded);
@Override public String toString() { return "Name: " + getName() + " id: " + getId() + " Priority: " + getTopologyPriority() + " Uptime: " + getUpTime() + " CPU: " + getTotalRequestedCpu() + " Memory: " + (getTotalRequestedMemOffHeap() + getTotalRequestedMemOnHeap()); }
LOG.debug("Not enough resources to schedule {}", td.getName()); List<TopologyDetails> reversedList = ImmutableList.copyOf(orderedTopologies).reverse(); boolean evictedSomething = false; LOG.debug("attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); int tdIndex = reversedList.indexOf(td); double cpuNeeded = td.getTotalRequestedCpu(); Collection<WorkerSlot> workersToEvict = workingState.getUsedSlotsByTopologyId(topologyEvict.getId()); LOG.debug("Evicting Topology {} with workers: {} from user {}", topologyEvict.getName(), workersToEvict, topologyEvict.getTopologySubmitter()); cpuNeeded -= getCpuUsed(evictAssignemnt);
private void moveTopology(TopologyDetails topo, Set<TopologyDetails> src, String srcName, Set<TopologyDetails> dest, String destName) { if (topo == null) { return; } LOG.debug("For User {} Moving topo {} from {} to {}", this.userId, topo.getName(), srcName, destName); if (!src.contains(topo)) { LOG.warn("Topo {} not in User: {} {} queue!", topo.getName(), this.userId, srcName); return; } if (dest.contains(topo)) { LOG.warn("Topo {} already in in User: {} {} queue!", topo.getName(), this.userId, destName); return; } src.remove(topo); dest.add(topo); }
public Topologies(Map<String, TopologyDetails> topologies) { if(topologies==null) topologies = new HashMap<>(); this.topologies = new HashMap<>(topologies.size()); this.topologies.putAll(topologies); this.nameToId = new HashMap<>(topologies.size()); for (Map.Entry<String, TopologyDetails> entry : topologies.entrySet()) { TopologyDetails topology = entry.getValue(); this.nameToId.put(topology.getName(), entry.getKey()); } }
public static String getWorkerPrefix(Map conf, TopologyDetails info) { Map topologyConf = getFullTopologyConfig(conf, info); String prefix = Optional.fromNullable((String) topologyConf.get(WORKER_NAME_PREFIX)).or(""); return String.format("%s%s%s", prefix, info.getName(), getWorkerPrefixDelimiter(conf)); }
LOG.error("Cannot determine user for topology {}. Will skip scheduling this topology", td.getName()); continue; LOG.debug("adding td: {} to pending queue", td.getName()); userMap.get(topologySubmitter).addTopologyToPendingQueue(td); } else { LOG.debug("adding td: {} to running queue with existing status: {}", td.getName(), cluster.getStatusMap().get(td.getId())); userMap.get(topologySubmitter).addTopologyToRunningQueue(td); if (cluster.getStatusMap().get(td.getId()) == null || cluster.getStatusMap().get(td.getId()).equals("")) {
private void evictTopology(TopologyDetails topologyEvict) { Collection<WorkerSlot> workersToEvict = this.cluster.getUsedSlotsByTopologyId(topologyEvict.getId()); User submitter = this.userMap.get(topologyEvict.getTopologySubmitter()); LOG.info("Evicting Topology {} with workers: {} from user {}", topologyEvict.getName(), workersToEvict, topologyEvict.getTopologySubmitter()); this.nodes.freeSlots(workersToEvict); submitter.moveTopoFromRunningToPending(topologyEvict, this.cluster); }
Node n = findBestNode(allNodes); if (n == null) { LOG.error("No nodes to use to assign topology {}", td.getName()); break;
@Override public String toString() { return "Name: " + this.getName() + " id: " + this.getId() + " Priority: " + this.getTopologyPriority() + " Uptime: " + this.getUpTime() + " CPU: " + this.getTotalRequestedCpu() + " Memory: " + (this.getTotalRequestedMemOffHeap() + this.getTotalRequestedMemOnHeap()); }
public void scheduleTopology(TopologyDetails td) { User topologySubmitter = this.schedulingState.userMap.get(td.getTopologySubmitter()); if (this.schedulingState.cluster.getUnassignedExecutors(td).size() > 0) { LOG.debug("/********Scheduling topology {} from User {}************/", td.getName(), topologySubmitter); } catch (RuntimeException e) { LOG.error("failed to create instance of IStrategy: {} with error: {}! Topology {} will not be scheduled.", td.getName(), td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY), e.getMessage()); topologySubmitter = cleanup(schedulingState, td); topologySubmitter.moveTopoFromPendingToInvalid(td); } catch (Exception ex) { LOG.error(String.format("Exception thrown when running strategy %s to schedule topology %s. Topology will not be scheduled!" , rasStrategy.getClass().getName(), td.getName()), ex); topologySubmitter = cleanup(schedulingState, td); topologySubmitter.moveTopoFromPendingToInvalid(td); } catch (Exception ex) { LOG.error(String.format("Exception thrown when running eviction strategy %s to schedule topology %s. No evictions will be done! Error: %s" , evictionStrategy.getClass().getName(), td.getName(), ex.getClass().getName()), ex); topologySubmitter = cleanup(schedulingState, td); topologySubmitter.moveTopoFromPendingToAttempted(td); LOG.warn("Scheduling results returned from topology {} is not vaild! Topology with be ignored.", td.getName()); topologySubmitter = cleanup(schedulingState, td); topologySubmitter.moveTopoFromPendingToInvalid(td, this.schedulingState.cluster); LOG.warn("Topology {} is already fully scheduled!", td.getName()); topologySubmitter.moveTopoFromPendingToRunning(td); if (this.schedulingState.cluster.getStatusMap().get(td.getId()) == null || this.schedulingState.cluster.getStatusMap().get(td.getId()).equals("")) {
@Override public boolean makeSpaceForTopo(TopologyDetails td) { LOG.debug("attempting to make space for topo {} from user {}", td.getName(), td.getTopologySubmitter()); User submitter = this.userMap.get(td.getTopologySubmitter()); if (submitter.getCPUResourceGuaranteed() == null || submitter.getMemoryResourceGuaranteed() == null