Map<String, SupervisorResources> ret = new HashMap<>(); for (SupervisorDetails sd : supervisors.values()) { ret.put(sd.getId(), new SupervisorResources(sd.getTotalMemory(), sd.getTotalCpu(), 0, 0)); SupervisorResources sr = ret.get(id); if (sr == null) { sr = new SupervisorResources(0, 0, 0, 0); sr = sr.add(entry.getValue()); ret.put(id, sr); SupervisorResources sr = ret.get(id); if (sr == null) { sr = new SupervisorResources(0, 0, 0, 0); sr = sr.addMem(entry.getValue()); ret.put(id, sr);
private boolean isFragmented(SupervisorResources supervisorResources) { double minMemory = ObjectReader.getDouble(conf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), 256.0) + ObjectReader.getDouble(conf.get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB), 128.0); double minCPU = ObjectReader.getDouble(conf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), 50.0) + ObjectReader.getDouble(conf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT), 50.0); return minMemory > supervisorResources.getAvailableMem() || minCPU > supervisorResources.getAvailableCpu(); }
LOG.info("Fragmentation after scheduling is: {} MB, {} PCore CPUs", fragmentedMemory(), fragmentedCpu()); nodeIdToResources.get().forEach((id, node) -> { final double availableMem = node.getAvailableMem(); if (availableMem < 0) { LOG.warn("Memory over-scheduled on {}", id, availableMem); final double availableCpu = node.getAvailableCpu(); if (availableCpu < 0) { LOG.warn("CPU over-scheduled on {}", id, availableCpu); "Node Id: {} Total Mem: {}, Used Mem: {}, Available Mem: {}, Total CPU: {}, Used " + "CPU: {}, Available CPU: {}, fragmented: {}", id, node.getTotalMem(), node.getUsedMem(), availableMem, node.getTotalCpu(), node.getUsedCpu(), availableCpu, isFragmented(node)); });
String supervisorId = entry.getKey(); SupervisorResources resources = entry.getValue(); assertTrue(supervisorId, resources.getTotalCpu() >= resources.getUsedCpu()); assertTrue(supervisorId, resources.getTotalMem() >= resources.getUsedMem());
SupervisorResources resources = nodeIdToResources.get().get(supervisorId); if (resources != null) { ret.set_used_mem(resources.getUsedMem()); ret.set_used_cpu(resources.getUsedCpu()); if (isFragmented(resources)) { final double availableCpu = resources.getAvailableCpu(); if (availableCpu < 0) { LOG.warn("Negative fragmented CPU on {}", supervisorId); final double availableMem = resources.getAvailableMem(); if (availableMem < 0) { LOG.warn("Negative fragmented Mem on {}", supervisorId);
public SupervisorResources addMem(Double value) { return new SupervisorResources(totalMem, totalCpu, usedMem + value, usedCpu); } }
String supervisorId = entry.getKey(); SupervisorResources resources = entry.getValue(); assertTrue(supervisorId, resources.getTotalCpu() >= resources.getUsedCpu()); assertTrue(supervisorId, resources.getTotalMem() >= resources.getUsedMem());
SupervisorResources add(WorkerResources wr) { return new SupervisorResources( totalMem, totalCpu, usedMem + wr.get_mem_off_heap() + wr.get_mem_on_heap(), usedCpu + wr.get_cpu()); }
.mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableMem(), 0)) .sum()); metricsRegistry.registerGauge("nimbus:available-cpu-non-negative", () -> nodeIdToResources.get().values() .parallelStream() .mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableCpu(), 0)) .sum()); metricsRegistry.registerGauge("nimbus:total-memory", () -> nodeIdToResources.get().values()