private void computeReducerTimeStatsPerJob(RunningJob rj) throws IOException { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(0); List<Integer> reducersRunTimes = new ArrayList<Integer>(); for (TaskCompletionEvent taskCompletion : taskCompletions) { if (!taskCompletion.isMapTask()) { reducersRunTimes.add(new Integer(taskCompletion.getTaskRunTime())); } } // Compute the reducers run time statistics for the job ReducerTimeStatsPerJob reducerTimeStatsPerJob = new ReducerTimeStatsPerJob(reducersRunTimes); // Adding the reducers run time statistics for the job in the QueryPlan this.task.getQueryPlan().getReducerTimeStatsPerJobList().add(reducerTimeStatsPerJob); return; }
private void computeReducerTimeStatsPerJob(RunningJob rj) throws IOException { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(0); List<Integer> reducersRunTimes = new ArrayList<Integer>(); for (TaskCompletionEvent taskCompletion : taskCompletions) { if (!taskCompletion.isMapTask()) { reducersRunTimes.add(new Integer(taskCompletion.getTaskRunTime())); } } // Compute the reducers run time statistics for the job ReducerTimeStatsPerJob reducerTimeStatsPerJob = new ReducerTimeStatsPerJob(reducersRunTimes); // Adding the reducers run time statistics for the job in the QueryPlan this.task.getQueryPlan().getReducerTimeStatsPerJobList().add(reducerTimeStatsPerJob); return; }
@Override public MapTaskCompletionEventsUpdate getMapCompletionEvents(JobID jobId, int fromIndex, int maxLocs, TaskAttemptID id) throws IOException { TaskCompletionEvent[] recentEvents = jobTracker.getTaskCompletionEvents(jobId, totalEventsFetched, Integer.MAX_VALUE); totalEventsFetched += recentEvents.length; for (TaskCompletionEvent event : recentEvents) { if (event.isMapTask()) { mapEventFetched.add(event); } } int toIndex = fromIndex + maxLocs; toIndex = toIndex > mapEventFetched.size() ? mapEventFetched.size() : toIndex; TaskCompletionEvent[] result = mapEventFetched.subList(fromIndex, toIndex). toArray(new TaskCompletionEvent[toIndex - fromIndex]); return new MapTaskCompletionEventsUpdate(result, false); } }
/** Queries the job tracker for a set of outputs ready to be copied * @param fromEventId the first event ID we want to start from, this is * modified by the call to this method * @param jobClient the job tracker * @return a set of locations to copy outputs from * @throws IOException */ private List<TaskCompletionEvent> queryJobTracker(IntWritable fromEventId, JobID jobId, InterTrackerProtocol jobClient) throws IOException { TaskCompletionEvent t[] = jobClient.getTaskCompletionEvents( jobId, fromEventId.get(), probe_sample_size); //we are interested in map task completion events only. So store //only those List <TaskCompletionEvent> recentMapEvents = new ArrayList<TaskCompletionEvent>(); for (int i = 0; i < t.length; i++) { if (t[i].isMapTask()) { recentMapEvents.add(t[i]); } } fromEventId.set(fromEventId.get() + t.length); return recentMapEvents; }
private void computeReducerTimeStatsPerJob(RunningJob rj) throws IOException { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(0); List<Integer> reducersRunTimes = new ArrayList<Integer>(); for (TaskCompletionEvent taskCompletion : taskCompletions) { if (!taskCompletion.isMapTask()) { reducersRunTimes.add(new Integer(taskCompletion.getTaskRunTime())); } } // Compute the reducers run time statistics for the job ReducerTimeStatsPerJob reducerTimeStatsPerJob = new ReducerTimeStatsPerJob(reducersRunTimes); // Adding the reducers run time statistics for the job in the QueryPlan this.task.getQueryPlan().getReducerTimeStatsPerJobList().add(reducerTimeStatsPerJob); return; }
@Override public boolean equals(Object o) { if(o == null) return false; if(o.getClass().equals(TaskCompletionEvent.class)) { TaskCompletionEvent event = (TaskCompletionEvent) o; return this.isMap == event.isMapTask() && this.eventId == event.getEventId() && this.idWithinJob == event.idWithinJob() && this.status.equals(event.getTaskStatus()) && this.taskId.equals(event.getTaskAttemptId()) && this.taskRunTime == event.getTaskRunTime() && this.taskTrackerHttp.equals(event.getTaskTrackerHttp()); } return false; }
@Override public boolean equals(Object o) { if(o == null) return false; if(o.getClass().equals(TaskCompletionEvent.class)) { TaskCompletionEvent event = (TaskCompletionEvent) o; return this.isMap == event.isMapTask() && this.eventId == event.getEventId() && this.idWithinJob == event.idWithinJob() && this.status.equals(event.getTaskStatus()) && this.taskId.equals(event.getTaskAttemptId()) && this.taskRunTime == event.getTaskRunTime() && this.taskTrackerHttp.equals(event.getTaskTrackerHttp()); } return false; }
new TaskCompletionEvent(mapEvent.getEventId(), mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
new TaskCompletionEvent(mapEvent.getEventId(), mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
new TaskCompletionEvent(mapEvent.getEventId(), mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));