Refine search
private void getTaskInfos() throws IOException, MalformedURLException { int startIndex = 0; while (true) { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex); String taskId = t.getTaskAttemptId().getTaskID().toString(); String jobId = t.getTaskAttemptId().getJobID().toString(); if (firstError) { console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId); conf, t.getTaskTrackerHttp(), t.getTaskId()); if (taskAttemptLogUrl != null) { ti.getLogUrls().add(taskAttemptLogUrl); if (t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) { String[] diags = rj.getTaskDiagnostics(t.getTaskAttemptId()); ti.setDiagnosticMesgs(diags); if (ti.getErrorCode() == 0) {
/** * test deprecated methods of TaskCompletionEvent */ @SuppressWarnings("deprecation") @Test (timeout=5000) public void testTaskCompletionEvent() { TaskAttemptID taid = new TaskAttemptID("001", 1, TaskType.REDUCE, 2, 3); TaskCompletionEvent template = new TaskCompletionEvent(12, taid, 13, true, Status.SUCCEEDED, "httptracker"); TaskCompletionEvent testEl = TaskCompletionEvent.downgrade(template); testEl.setTaskAttemptId(taid); testEl.setTaskTrackerHttp("httpTracker"); testEl.setTaskId("attempt_001_0001_m_000002_04"); assertEquals("attempt_001_0001_m_000002_4",testEl.getTaskId()); testEl.setTaskStatus(Status.OBSOLETE); assertEquals(Status.OBSOLETE.toString(), testEl.getStatus().toString()); testEl.setTaskRunTime(20); assertEquals(testEl.getTaskRunTime(), 20); testEl.setEventId(16); assertEquals(testEl.getEventId(), 16); }
private void computeReducerTimeStatsPerJob(RunningJob rj) throws IOException { TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(0); List<Integer> reducersRunTimes = new ArrayList<Integer>(); for (TaskCompletionEvent taskCompletion : taskCompletions) { if (!taskCompletion.isMapTask()) { reducersRunTimes.add(new Integer(taskCompletion.getTaskRunTime())); } } // Compute the reducers run time statistics for the job ReducerTimeStatsPerJob reducerTimeStatsPerJob = new ReducerTimeStatsPerJob(reducersRunTimes); // Adding the reducers run time statistics for the job in the QueryPlan this.task.getQueryPlan().getReducerTimeStatsPerJobList().add(reducerTimeStatsPerJob); return; }
eventCounter += events.length; for(TaskCompletionEvent event : events){ TaskCompletionEvent.Status status = event.getTaskStatus(); if (profiling && (status == TaskCompletionEvent.Status.SUCCEEDED || status == TaskCompletionEvent.Status.FAILED) && (event.isMap ? mapRanges : reduceRanges). isIncluded(event.idWithinJob())) { break; case SUCCEEDED: if (event.getTaskStatus() == TaskCompletionEvent.Status.SUCCEEDED){ LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED){ LOG.info(event.toString()); TaskAttemptID taskId = event.getTaskAttemptId(); String[] taskDiagnostics = jc.jobSubmitClient.getTaskDiagnostics(taskId); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); if (event.getTaskStatus() == TaskCompletionEvent.Status.KILLED){ LOG.info(event.toString());
@Override public void resolve(TaskCompletionEvent event) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(reduceId, event.getTaskTrackerHttp()); addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); maxMapRuntime = Math.max(maxMapRuntime, event.getTaskRunTime()); break; case FAILED: case KILLED: case OBSOLETE: obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } }
JobClient jClient = new JobClient(conf); JobStatus[] jStatus = jClient.getAllJobs(); JobStatus js = jStatus[0]; // Our only job RunningJob rj = jClient.getJob(js.getJobID()); TaskCompletionEvent[] taskComplEvents = rj.getTaskCompletionEvents(0); rj.getTaskDiagnostics(tce.getTaskAttemptId());
public void testPersistency() throws Exception { Properties config = new Properties(); config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "true"); config.setProperty(JTConfig.JT_PERSIST_JOBSTATUS_HOURS, "1"); startCluster(true, config); JobID jobId = runJob(); JobClient jc = new JobClient(createJobConf()); RunningJob rj0 = jc.getJob(jobId); assertNotNull(rj0); boolean sucessfull0 = rj0.isSuccessful(); String jobName0 = rj0.getJobName(); Counters counters0 = rj0.getCounters(); TaskCompletionEvent[] events0 = rj0.getTaskCompletionEvents(0); stopCluster(); startCluster(false, config); jc = new JobClient(createJobConf()); RunningJob rj1 = jc.getJob(jobId); assertNotNull(rj1); assertEquals(sucessfull0, rj1.isSuccessful()); assertEquals(jobName0, rj0.getJobName()); assertEquals(counters0.size(), rj1.getCounters().size()); TaskCompletionEvent[] events1 = rj1.getTaskCompletionEvents(0); assertEquals(events0.length, events1.length); for (int i = 0; i < events0.length; i++) { assertEquals(events0[i].getTaskAttemptId(), events1[i].getTaskAttemptId()); assertEquals(events0[i].getTaskStatus(), events1[i].getTaskStatus()); } }
eventCounter += events.length; for(TaskCompletionEvent event : events){ TaskCompletionEvent.Status status = event.getTaskStatus(); if (profiling && (status == TaskCompletionEvent.Status.SUCCEEDED || status == TaskCompletionEvent.Status.FAILED) && (event.isMap ? mapRanges : reduceRanges). isIncluded(event.idWithinJob())) { downloadProfile(event); break; case SUCCEEDED: if (event.getTaskStatus() == TaskCompletionEvent.Status.SUCCEEDED){ LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED){ LOG.info(event.toString()); TaskAttemptID taskId = event.getTaskAttemptId(); String[] taskDiagnostics = jobSubmitClient.getTaskDiagnostics(taskId); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); if (event.getTaskStatus() == TaskCompletionEvent.Status.KILLED){ LOG.info(event.toString());
JobClient client = new JobClient(new JobConf(job.getConfiguration())); for (TaskReport task : client.getMapTaskReports(tasks[0].getTaskAttemptId().getJobID())) { if (task.getCurrentStatus().equals(TIPStatus.FAILED)) { for (String s : task.getDiagnostics()) {
boolean success = true; JobClient jobClient = new JobClient(this.conf); TaskReport[] setupReports = jobClient.getSetupTaskReports(oldJobId); if (setupReports.length > 0) for (TaskReport report : jobClient.getSetupTaskReports(oldJobId)) for (TaskCompletionEvent event : getTaskCompletionEvents(jobClient,oldJobId)) String type = taskIdToType.get(event.getTaskAttemptId().getTaskID().toString()); String status = event.getTaskStatus().toString();
job.mapAttemptCompletionEvents.get(mapCompletionIdx); job.mapAttemptCompletionEvents.set(mapCompletionIdx, new TaskCompletionEvent(mapEvent.getEventId(), mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
@Override public boolean equals(Object o) { if(o == null) return false; if(o.getClass().equals(TaskCompletionEvent.class)) { TaskCompletionEvent event = (TaskCompletionEvent) o; return this.isMap == event.isMapTask() && this.eventId == event.getEventId() && this.idWithinJob == event.idWithinJob() && this.status.equals(event.getTaskStatus()) && this.taskId.equals(event.getTaskAttemptId()) && this.taskRunTime == event.getTaskRunTime() && this.taskTrackerHttp.equals(event.getTaskTrackerHttp()); } return false; }
/** * Returns task id. * @return task id * @deprecated use {@link #getTaskAttemptId()} instead. */ @Deprecated public String getTaskId() { return getTaskAttemptId().toString(); }
&& (searchUntilFound || System.currentTimeMillis() - start < MILLIS_TO_SEARCH)) { for (TaskCompletionEvent event : events) { if (event.getTaskStatus() == TaskCompletionEvent.Status.FAILED) { failures.add(event); int num_queries = Math.min(FAILURES_TO_QUERY, failures.size()); for (int i = 0; i < num_queries; i++) { TaskAttemptID taskAttemptID = failures.get(i).getTaskAttemptId(); String taskAttemptHost = failures.get(i).getTaskTrackerHttp(); String[] fails = job.getTaskDiagnostics(taskAttemptID); for (String failure : fails) { jobFailures.add(new TaskFailure(taskAttemptID.toString(), taskAttemptHost, failure));
taskEvent = new TaskCompletionEvent( taskCompletionEventTracker, taskid, httpTaskLogLocation ); taskEvent.setTaskRunTime((int)(status.getFinishTime() - status.getStartTime())); tip.setSuccessEventNumber(taskCompletionEventTracker); TaskCompletionEvent t = this.taskCompletionEvents.get(eventNumber); if (t.getTaskAttemptId().equals(taskid)) t.setTaskStatus(TaskCompletionEvent.Status.OBSOLETE); taskCompletionStatus = TaskCompletionEvent.Status.TIPFAILED; taskEvent = new TaskCompletionEvent(taskCompletionEventTracker, taskid, tip.idWithinJob(),
/** * List the events for the given job * @param jobId the job id for the job's events to list * @throws IOException */ private void listEvents(JobID jobId, int fromEventId, int numEvents) throws IOException { TaskCompletionEvent[] events = jobSubmitClient.getTaskCompletionEvents(jobId, fromEventId, numEvents); System.out.println("Task completion events for " + jobId); System.out.println("Number of events (from " + fromEventId + ") are: " + events.length); for(TaskCompletionEvent event: events) { System.out.println(event.getTaskStatus() + " " + event.getTaskAttemptId() + " " + getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp())); } }
public void readFields(DataInput in) throws IOException { reset = in.readBoolean(); events = new TaskCompletionEvent[in.readInt()]; for (int i = 0; i < events.length; ++i) { events[i] = new TaskCompletionEvent(); events[i].readFields(in); } } }
@Override public String[] getTaskJobIDs(TaskCompletionEvent t) { TaskID tid = t.getTaskAttemptId().getTaskID(); ret[0] = tid.toString(); ret[1] = tid.getJobID().toString(); return ret; }
private static void downloadProfile(TaskCompletionEvent e ) throws IOException { URLConnection connection = new URL(getTaskLogURL(e.getTaskAttemptId(), e.getTaskTrackerHttp()) + "&filter=profile").openConnection(); InputStream in = connection.getInputStream(); OutputStream out = new FileOutputStream(e.getTaskAttemptId() + ".profile"); IOUtils.copyBytes(in, out, 64 * 1024, true); }
private MapTaskCompletionEventsUpdate getMockedCompletionEventsUpdate( int startIdx, int numEvents) { ArrayList<TaskCompletionEvent> tceList = new ArrayList<TaskCompletionEvent>(numEvents); for (int i = 0; i < numEvents; ++i) { int eventIdx = startIdx + i; TaskCompletionEvent tce = new TaskCompletionEvent(eventIdx, new TaskAttemptID("12345", 1, TaskType.MAP, eventIdx, 0), eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED, "http://somehost:8888"); tceList.add(tce); } TaskCompletionEvent[] events = {}; return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false); }