public static JobletProfile create(DataInput dis) throws IOException { JobletProfile jobletProfile = new JobletProfile(); jobletProfile.readFields(dis); return jobletProfile; }
public static JobProfile create(DataInput dis) throws IOException { JobProfile jobProfile = new JobProfile(); jobProfile.readFields(dis); return jobProfile; }
public static PartitionProfile create(DataInput dis) throws IOException { PartitionProfile partitionProfile = new PartitionProfile(); partitionProfile.readFields(dis); return partitionProfile; }
@Override protected void performEvent(TaskAttempt ta) { IJobManager jobManager = ccs.getJobManager(); JobRun run = jobManager.get(jobId); if (run == null) { return; } if (statistics != null) { JobProfile jobProfile = run.getJobProfile(); Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles(); JobletProfile jobletProfile = jobletProfiles.get(nodeId); if (jobletProfile == null) { jobletProfile = new JobletProfile(nodeId); jobletProfiles.put(nodeId, jobletProfile); } jobletProfile.getTaskProfiles().put(taId, statistics); } run.getExecutor().notifyTaskComplete(ta); }
private void updateJobStats(JobId jobId, Stats stats) { final IJobManager jobManager = ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobManager(); final JobRun run = jobManager.get(jobId); if (run == null || run.getStatus() != JobStatus.TERMINATED) { return; } final JobProfile jobProfile = run.getJobProfile(); final Collection<JobletProfile> jobletProfiles = jobProfile.getJobletProfiles().values(); long processedObjects = 0; for (JobletProfile jp : jobletProfiles) { final Collection<TaskProfile> jobletTasksProfile = jp.getTaskProfiles().values(); for (TaskProfile tp : jobletTasksProfile) { processedObjects += tp.getStatsCollector().getAggregatedStats().getTupleCounter().get(); } } stats.setProcessedObjects(processedObjects); }
public void dumpProfile(JobletProfile jProfile) { Map<String, Long> counters = jProfile.getCounters(); counterMap.forEach((key, value) -> counters.put(key, value.get())); for (Task task : taskMap.values()) { TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(), new Hashtable<>(task.getPartitionSendProfile()), new StatsCollector()); task.dumpProfile(taskProfile); jProfile.getTaskProfiles().put(task.getTaskAttemptId(), taskProfile); } }
public static TaskProfile create(DataInput dis) throws IOException { TaskProfile taskProfile = new TaskProfile(); taskProfile.readFields(dis); return taskProfile; }
public void setPartitionSendProfile(PartitionProfile profile) { partitionSendProfile.put(profile.getPartitionId(), profile); }
@Override public void readFields(DataInput input) throws IOException { super.readFields(input); nodeId = input.readUTF(); int size = input.readInt(); taskProfiles = new HashMap<>(); for (int i = 0; i < size; i++) { TaskAttemptId key = TaskAttemptId.create(input); TaskProfile value = TaskProfile.create(input); taskProfiles.put(key, value); } }
@Override public void writeFields(DataOutput output) throws IOException { super.writeFields(output); output.writeUTF(nodeId); output.writeInt(taskProfiles.size()); for (Entry<TaskAttemptId, TaskProfile> entry : taskProfiles.entrySet()) { entry.getKey().writeFields(output); entry.getValue().writeFields(output); } } }
@Override public ObjectNode toJSON() { ObjectMapper om = new ObjectMapper(); ObjectNode json = om.createObjectNode(); json.put("node-id", nodeId); populateCounters(json); ArrayNode tasks = om.createArrayNode(); for (TaskProfile p : taskProfiles.values()) { tasks.add(p.toJSON()); } json.set("tasks", tasks); return json; }
@Override public ObjectNode toJSON() { ObjectMapper om = new ObjectMapper(); ObjectNode json = om.createObjectNode(); json.put("job-id", jobId.toString()); populateCounters(json); ArrayNode jobletsArray = om.createArrayNode(); for (JobletProfile p : jobletProfiles.values()) { jobletsArray.add(p.toJSON()); } json.set("joblets", jobletsArray); return json; }
@Override public void readFields(DataInput input) throws IOException { super.readFields(input); taskAttemptId = TaskAttemptId.create(input); int size = input.readInt(); partitionSendProfile = new HashMap<>(); for (int i = 0; i < size; i++) { PartitionId key = PartitionId.create(input); PartitionProfile value = PartitionProfile.create(input); partitionSendProfile.put(key, value); } statsCollector = StatsCollector.create(input); }
@Override public void writeFields(DataOutput output) throws IOException { super.writeFields(output); taskAttemptId.writeFields(output); output.writeInt(partitionSendProfile.size()); for (Entry<PartitionId, PartitionProfile> entry : partitionSendProfile.entrySet()) { entry.getKey().writeFields(output); entry.getValue().writeFields(output); } statsCollector.writeFields(output); } }
@Override public void writeFields(DataOutput output) throws IOException { jobId.writeFields(output); output.writeInt(jobletProfiles.size()); for (Entry<String, JobletProfile> entry : jobletProfiles.entrySet()) { output.writeUTF(entry.getKey()); entry.getValue().writeFields(output); } } }
public synchronized void dumpProfile(TaskProfile tProfile) { Map<String, Long> dumpMap = tProfile.getCounters(); for (Counter c : counterMap.values()) { dumpMap.put(c.getName(), c.get()); } }
public static void serialize(OutputStream out, Object object) throws Exception { NotifyTaskCompleteFunction fn = (NotifyTaskCompleteFunction) object; DataOutputStream dos = new DataOutputStream(out); fn.jobId.writeFields(dos); dos.writeUTF(fn.nodeId); fn.taskId.writeFields(dos); fn.statistics.writeFields(dos); } }