private void updateHistogram(ClusterSummary newSummary) { for (NimbusSummary nimbusSummary : newSummary.get_nimbuses()) { nimbusUptime.update(nimbusSummary.get_uptime_secs()); } for (SupervisorSummary summary : newSummary.get_supervisors()) { supervisorsUptime.update(summary.get_uptime_secs()); supervisorsNumWorkers.update(summary.get_num_workers()); supervisorsNumUsedWorkers.update(summary.get_num_used_workers()); supervisorsUsedMem.update(Math.round(summary.get_used_mem())); supervisorsUsedCpu.update(Math.round(summary.get_used_cpu())); supervisorsFragmentedMem.update(Math.round(summary.get_fragmented_mem())); supervisorsFragmentedCpu.update(Math.round(summary.get_fragmented_cpu())); } for (TopologySummary summary : newSummary.get_topologies()) { topologiesNumTasks.update(summary.get_num_tasks()); topologiesNumExecutors.update(summary.get_num_executors()); topologiesNumWorker.update(summary.get_num_workers()); topologiesUptime.update(summary.get_uptime_secs()); topologiesReplicationCount.update(summary.get_replication_count()); topologiesRequestedMemOnHeap.update(Math.round(summary.get_requested_memonheap())); topologiesRequestedMemOffHeap.update(Math.round(summary.get_requested_memoffheap())); topologiesRequestedCpu.update(Math.round(summary.get_requested_cpu())); topologiesAssignedMemOnHeap.update(Math.round(summary.get_assigned_memonheap())); topologiesAssignedMemOffHeap.update(Math.round(summary.get_assigned_memoffheap())); topologiesAssignedCpu.update(Math.round(summary.get_assigned_cpu())); } }
result.put("tasksTotal", topologySummary.get_num_tasks()); result.put("workersTotal", topologySummary.get_num_workers()); result.put("executorsTotal", topologySummary.get_num_executors()); result.put("replicationCount", topologySummary.get_replication_count()); result.put("schedulerInfo", topologySummary.get_sched_status());
private static List<DataPoint> extractClusterMetrics(ClusterSummary summ) { List<DataPoint> ret = new ArrayList<>(); ret.add(new DataPoint("supervisors", summ.get_supervisors_size())); ret.add(new DataPoint("topologies", summ.get_topologies_size())); int totalSlots = 0; int usedSlots = 0; for (SupervisorSummary sup : summ.get_supervisors()) { usedSlots += sup.get_num_used_workers(); totalSlots += sup.get_num_workers(); } ret.add(new DataPoint("slotsTotal", totalSlots)); ret.add(new DataPoint("slotsUsed", usedSlots)); ret.add(new DataPoint("slotsFree", totalSlots - usedSlots)); int totalExecutors = 0; int totalTasks = 0; for (TopologySummary topo : summ.get_topologies()) { totalExecutors += topo.get_num_executors(); totalTasks += topo.get_num_tasks(); } ret.add(new DataPoint("executorsTotal", totalExecutors)); ret.add(new DataPoint("tasksTotal", totalTasks)); return ret; }
public static MetricsSample factory(Nimbus.Iface client, String topologyName) throws Exception { // "************ Sampling Metrics ***************** ClusterSummary clusterSummary = client.getClusterInfo(); // get topology info TopologySummary topSummary = getTopologySummary(clusterSummary, topologyName); int topologyExecutors = topSummary.get_num_executors(); int topologyWorkers = topSummary.get_num_workers(); int topologyTasks = topSummary.get_num_tasks(); TopologyInfo topInfo = client.getTopologyInfo(topSummary.get_id()); MetricsSample sample = getMetricsSample(topInfo); sample.numWorkers = topologyWorkers; sample.numExecutors = topologyExecutors; sample.numTasks = topologyTasks; return sample; }
return get_num_executors();
private Pair<Integer, Integer> getExecutorNum(List<TopologySummary> summaries) { int currentExecutorNum = 0; int currentWorkerNum = 0; for (TopologySummary summary : summaries) { if (summary.get_name().equals(getAppName())) { currentExecutorNum = summary.get_num_executors(); currentWorkerNum = summary.get_num_workers(); } } return new Pair<Integer, Integer>(currentExecutorNum, currentWorkerNum); }