null, null, 0, 0, 0); ResourceCalculatorProcessTree pt = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(this.containerProcessId, this.processTreeClass, conf); processTreeInfo.setPid(this.containerProcessId); processTreeInfo.setProcessTree(pt);
long vmem = 0; if (processTree != null) { rss = processTree.getRssMemorySize(); vmem = processTree.getVirtualMemorySize();
try { while (true) { pTree.updateProcessTree(); Long time = (new Date()).getTime(); double currentPmemUsage = 0.0; } catch (NoSuchMethodException e) { LOG.debug("current hadoop version don't have the method getRssMemorySize of Class " + pTree.getClass().toString() + ". For More Detail: " + e); currentPmemUsage = Double.parseDouble(df.format(pTree.getCumulativeRssmem() / 1024.0 / 1024.0 / 1024.0));
private LlapDaemonJvmMetrics(String displayName, String sessionId, final Configuration conf) { this.name = displayName; this.sessionId = sessionId; Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null, ResourceCalculatorProcessTree.class); this.processTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("" + daemonPid, clazz, conf); if (processTree != null) { this.processTree.setConf(conf); } this.registry = new MetricsRegistry("LlapDaemonJvmRegistry"); this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId); }
ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf); ptInfo.setPid(pId); ptInfo.setProcessTree(pt); + " ContainerId = " + containerId); ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree(); pTree.updateProcessTree(); // update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); float cpuUsagePercentPerCore = pTree.getCpuUsagePercent(); float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors(); long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1); long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1); long vmemLimit = ptInfo.getVmemLimit(); long pmemLimit = ptInfo.getPmemLimit(); if (!pTree.checkPidPgrpidForMatch()) { LOG.error("Killed container process with PID " + pId + " but it is not a process group leader.");
/** * Update resource information counters */ void updateResourceCounters() { // Update generic resource counters updateHeapUsageCounter(); // Updating resources specified in ResourceCalculatorPlugin if (pTree == null) { return; } pTree.updateProcessTree(); long cpuTime = pTree.getCumulativeCpuTime(); long pMem = pTree.getRssMemorySize(); long vMem = pTree.getVirtualMemorySize(); // Remove the CPU time consumed previously by JVM reuse cpuTime -= initCpuCumulativeTime; tezCounters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime); tezCounters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem); tezCounters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem); }
null, ResourceCalculatorProcessTree.class); pTree = ResourceCalculatorProcessTree .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf); LOG.info(" Using ResourceCalculatorProcessTree : " + pTree); if (pTree != null) { pTree.updateProcessTree(); initCpuCumulativeTime = pTree.getCumulativeCpuTime();
pTree.updateProcessTree(); // update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); float cpuUsagePercentPerCore = pTree.getCpuUsagePercent(); if (cpuUsagePercentPerCore < 0) {
/** * Get the virtual memory used by all the processes in the * process-tree. * * @return virtual memory used by the process-tree in bytes, * {@link #UNAVAILABLE} if it cannot be calculated. */ public long getVirtualMemorySize() { return getVirtualMemorySize(0); }
@Test public void testCreatedInstanceConfigured() { ResourceCalculatorProcessTree tree; Configuration conf = new Configuration(); tree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("1", EmptyProcessTree.class, conf); assertNotNull(tree); assertThat(tree.getConf(), sameInstance(conf)); } }
long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1); long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1); if (isVmemCheckEnabled() && isProcessTreeOverLimit(containerId.toString(), if (!pTree.checkPidPgrpidForMatch()) { LOG.error("Killed container process with PID " + pId + " but it is not a process group leader.");
private void recordInitialCpuStats() { if (pTree != null) { pTree.updateProcessTree(); initCpuCumulativeTime = pTree.getCumulativeCpuTime(); } }
/** * Get the resident set size (rss) memory used by all the processes * in the process-tree. * * @return rss memory used by the process-tree in bytes, * {@link #UNAVAILABLE} if it cannot be calculated. */ public long getRssMemorySize() { return getRssMemorySize(0); }
/** * Format string when memory limit has been exceeded. * @param memTypeExceeded type of memory * @param usageString general memory usage information string * @param pId process id * @param containerId container id * @param pTree process tree to dump full resource utilization graph * @return formatted resource usage information */ private String formatErrorMessage(String memTypeExceeded, String usageString, String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree, long delta) { return String.format("Container [pid=%s,containerID=%s] is " + "running %dB beyond the '%S' memory limit. ", pId, containerId, delta, memTypeExceeded) + "Current usage: " + usageString + ". Killing container.\n" + "Dump of the process-tree for " + containerId + " :\n" + pTree.getProcessTreeDump(); }
float cpuUsagePercentPerCore = pTree.getCpuUsagePercent(); float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors();
ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf); ptInfo.setPid(pId); ptInfo.setProcessTree(pt); + " ContainerId = " + containerId); ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree(); pTree.updateProcessTree(); // update process-tree long currentVmemUsage = pTree.getVirtualMemorySize(); long currentPmemUsage = pTree.getRssMemorySize(); float cpuUsagePercentPerCore = pTree.getCpuUsagePercent(); float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors(); long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1); long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1); long vmemLimit = ptInfo.getVmemLimit(); long pmemLimit = ptInfo.getPmemLimit(); if (!pTree.checkPidPgrpidForMatch()) { LOG.error("Killed container process with PID " + pId + " but it is not a process group leader.");
return; pTree.updateProcessTree(); long cpuTime = pTree.getCumulativeCpuTime(); long pMem = pTree.getRssMemorySize(); long vMem = pTree.getVirtualMemorySize();
null, ResourceCalculatorProcessTree.class); pTree = ResourceCalculatorProcessTree .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf); LOG.info(" Using ResourceCalculatorProcessTree : " + pTree); if (pTree != null) { pTree.updateProcessTree(); initCpuCumulativeTime = pTree.getCumulativeCpuTime();
/** * Get the virtual memory used by all the processes in the * process-tree. * * @return virtual memory used by the process-tree in bytes, * {@link #UNAVAILABLE} if it cannot be calculated. */ public long getVirtualMemorySize() { return getVirtualMemorySize(0); }
@Test public void testCreatedInstanceConfigured() { ResourceCalculatorProcessTree tree; Configuration conf = new Configuration(); tree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("1", EmptyProcessTree.class, conf); assertNotNull(tree); assertThat(tree.getConf(), sameInstance(conf)); } }