@Override public boolean equals(Object o) { if (!(o instanceof TopologyDetails)) { return false; } return (topologyId.equals(((TopologyDetails) o).getId())); } }
private static Map<String, TopologyDetails> mkMap(TopologyDetails[] details) { Map<String, TopologyDetails> ret = new HashMap<>(); for (TopologyDetails td : details) { if (ret.put(td.getId(), td) != null) { throw new IllegalArgumentException( "Cannot have multiple topologies with the id " + td.getId()); } } return ret; }
private Set<String> extractTopologyIds(List<TopologyDetails> topologies) { Set<String> ids = new HashSet<String>(); if (topologies != null && topologies.size() > 0) { for (TopologyDetails topology : topologies) { ids.add(topology.getId()); } } return ids; }
/** * set scheduler status for a topology. */ public void setStatus(TopologyDetails td, String statusMessage) { setStatus(td.getId(), statusMessage); }
public void markTopoUnsuccess(TopologyDetails topo, Cluster cluster) { unsuccess.add(topo); if (cluster != null) { cluster.setStatus(topo.getId(), "Scheduling Attempted but topology is invalid"); } }
private Map<String, Set<Set<ExecutorDetails>>> topologyWorkerSpecs(List<TopologyDetails> topologies) { Map<String, Set<Set<ExecutorDetails>>> workerSpecs = new HashMap<String, Set<Set<ExecutorDetails>>>(); for (TopologyDetails topology : topologies) { workerSpecs.put(topology.getId(), computeWorkerSpecs(topology)); } return workerSpecs; }
private Map<String, Map<Integer, Integer>> topologyMachineDistributions(List<TopologyDetails> isoTopologies) { Map<String, Map<Integer, Integer>> machineDistributions = new HashMap<String, Map<Integer, Integer>>(); for (TopologyDetails topology : isoTopologies) { machineDistributions.put(topology.getId(), machineDistribution(topology)); } return machineDistributions; }
@Override public int getAssignedNumWorkers(TopologyDetails topology) { SchedulerAssignment assignment = topology != null ? this.getAssignmentById(topology.getId()) : null; if (assignment == null) { return 0; } Set<WorkerSlot> slots = new HashSet<>(); slots.addAll(assignment.getExecutorToSlot().values()); return slots.size(); }
@Override public Map<String, TopologyResources> getTopologyResourcesMap() { Map<String, TopologyResources> ret = new HashMap<>(assignments.size()); for (TopologyDetails td : topologies.getTopologies()) { String topoId = td.getId(); SchedulerAssignmentImpl assignment = assignments.get(topoId); ret.put(topoId, new TopologyResources(td, assignment)); } return ret; }
private Topologies leftoverTopologies(Topologies topologies, Set<String> filterIds) { Collection<TopologyDetails> topos = topologies.getTopologies(); Map<String, TopologyDetails> leftoverTopologies = new HashMap<String, TopologyDetails>(); for (TopologyDetails topo : topos) { String id = topo.getId(); if (!filterIds.contains(id)) { leftoverTopologies.put(id, topo); } } return new Topologies(leftoverTopologies); }
private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message, Throwable t) { c.setStatus(td, message); String realMessage = td.getId() + " " + message; if (t != null) { LOG.error(realMessage, t); } else { LOG.error(realMessage); } u.markTopoUnsuccess(td); }
@Override public void addTopology(TopologyDetails td) { String topId = td.getId(); LOG.debug("Adding in Topology {}", topId); _tds.put(topId, td); SchedulerAssignment assignment = _cluster.getAssignmentById(topId); if (assignment != null) { for (WorkerSlot ws : assignment.getSlots()) { Node n = _nodeIdToNode.get(ws.getNodeId()); _nodes.add(n); } } }
@Override public Map<ExecutorDetails, String> getNeedsSchedulingExecutorToComponents( TopologyDetails topology) { Collection<ExecutorDetails> allExecutors = new HashSet<>(topology.getExecutors()); SchedulerAssignment assignment = assignments.get(topology.getId()); if (assignment != null) { allExecutors.removeAll(assignment.getExecutors()); } return topology.selectExecutorToComponent(allExecutors); }
public static void scheduleTopologiesEvenly(Topologies topologies, Cluster cluster) { for (TopologyDetails topology : cluster.needsSchedulingTopologies()) { String topologyId = topology.getId(); Map<ExecutorDetails, WorkerSlot> newAssignment = scheduleTopology(topology, cluster); Map<WorkerSlot, List<ExecutorDetails>> nodePortToExecutors = Utils.reverseMap(newAssignment); for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : nodePortToExecutors.entrySet()) { WorkerSlot nodePort = entry.getKey(); List<ExecutorDetails> executors = entry.getValue(); cluster.assign(nodePort, topologyId, executors); } } }
public static Topologies addTopologies(Topologies topos, TopologyDetails... details) { Map<String, TopologyDetails> topoMap = new HashMap<>(); for (TopologyDetails td : topos.getTopologies()) { topoMap.put(td.getId(), td); } for (TopologyDetails td : details) { if (topoMap.put(td.getId(), td) != null) { throw new IllegalArgumentException("Cannot have multiple topologies with id " + td.getId()); } } return new Topologies(topoMap); }
public double getCpuResourceUsedByUser(ISchedulingState cluster) { double sum = 0.0; for (TopologyDetails td : cluster.getTopologies().getTopologiesOwnedBy(userId)) { SchedulerAssignment assignment = cluster.getAssignmentById(td.getId()); if (assignment != null) { TopologyResources tr = new TopologyResources(td, assignment); sum += tr.getAssignedCpu(); } } return sum; }
public double getMemoryResourceUsedByUser(ISchedulingState cluster) { double sum = 0.0; for (TopologyDetails td : cluster.getTopologies().getTopologiesOwnedBy(userId)) { SchedulerAssignment assignment = cluster.getAssignmentById(td.getId()); if (assignment != null) { TopologyResources tr = new TopologyResources(td, assignment); sum += tr.getAssignedMemOnHeap() + tr.getAssignedMemOffHeap(); } } return sum; }
public static void assertTopologiesFullyScheduled(Cluster cluster, String... topoNames) { Topologies topologies = cluster.getTopologies(); for (String topoName : topoNames) { TopologyDetails td = topologies.getByName(topoName); assert (td != null) : topoName; String topoId = td.getId(); assertStatusSuccess(cluster, topoId); assert (cluster.getAssignmentById(topoId) != null) : topoName; assert (cluster.needsSchedulingRas(td) == false) : topoName; } }
public static void assertTopologiesNotScheduled(Cluster cluster, String... topoNames) { Topologies topologies = cluster.getTopologies(); for (String topoName : topoNames) { TopologyDetails td = topologies.getByName(topoName); assert (td != null) : topoName; String topoId = td.getId(); String status = cluster.getStatus(topoId); assert (status != null) : topoName; assert (!isStatusSuccess(status)) : topoName; assert (cluster.getAssignmentById(topoId) == null) : topoName; assert (cluster.needsSchedulingRas(td)) : topoName; } }