public static JobProfile create(DataInput dis) throws IOException { JobProfile jobProfile = new JobProfile(); jobProfile.readFields(dis); return jobProfile; }
@Override public ObjectNode toJSON() { ObjectMapper om = new ObjectMapper(); ObjectNode json = om.createObjectNode(); json.put("job-id", jobId.toString()); populateCounters(json); ArrayNode jobletsArray = om.createArrayNode(); for (JobletProfile p : jobletProfiles.values()) { jobletsArray.add(p.toJSON()); } json.set("joblets", jobletsArray); return json; }
@Override protected void doRun() throws Exception { List<JobProfile> profiles = new ArrayList<>(); Map<JobId, Joblet> jobletMap = ncs.getJobletMap(); jobletMap.values().stream().filter(ji -> ji.getJobId().getCcId().equals(ccId)) .forEach(ji -> profiles.add(new JobProfile(ji.getJobId()))); for (JobProfile jProfile : profiles) { Joblet ji; JobletProfile jobletProfile = new JobletProfile(ncs.getId()); ji = jobletMap.get(jProfile.getJobId()); if (ji != null) { ji.dumpProfile(jobletProfile); jProfile.getJobletProfiles().put(ncs.getId(), jobletProfile); } } fv.setValue(profiles); } }
@Override protected void performEvent(TaskAttempt ta) { IJobManager jobManager = ccs.getJobManager(); JobRun run = jobManager.get(jobId); if (run == null) { return; } if (statistics != null) { JobProfile jobProfile = run.getJobProfile(); Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); JobletProfile jobletProfile = jobletProfiles.get(nodeId); if (jobletProfile == null) { jobletProfile = new JobletProfile(nodeId); jobletProfiles.put(nodeId, jobletProfile); } jobletProfile.getTaskProfiles().put(taId, statistics); } run.getExecutor().notifyTaskComplete(ta); }
private JobRun(DeploymentId deploymentId, JobId jobId, Set<JobFlag> jobFlags, JobSpecification spec, ActivityClusterGraph acg) { this.deploymentId = deploymentId; this.jobId = jobId; this.jobFlags = jobFlags; this.spec = spec; this.acg = acg; activityClusterPlanMap = new HashMap<>(); pmm = new PartitionMatchMaker(); participatingNodeIds = new HashSet<>(); cleanupPendingNodeIds = new HashSet<>(); profile = new JobProfile(jobId); connectorPolicyMap = new HashMap<>(); operatorLocations = new HashMap<>(); createTime = System.currentTimeMillis(); }
result.set("profile", profile.toJSON());
private void updateJobStats(JobId jobId, Stats stats) { final IJobManager jobManager = ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobManager(); final JobRun run = jobManager.get(jobId); if (run == null || run.getStatus() != JobStatus.TERMINATED) { return; } final JobProfile jobProfile = run.getJobProfile(); final Collection<JobletProfile> jobletProfiles = jobProfile.getJobletProfiles().values(); long processedObjects = 0; for (JobletProfile jp : jobletProfiles) { final Collection<TaskProfile> jobletTasksProfile = jp.getTaskProfiles().values(); for (TaskProfile tp : jobletTasksProfile) { processedObjects += tp.getStatsCollector().getAggregatedStats().getTupleCounter().get(); } } stats.setProcessedObjects(processedObjects); }