conf, t.getTaskTrackerHttp(), t.getTaskId()); if (taskAttemptLogUrl != null) { ti.getLogUrls().add(taskAttemptLogUrl);
conf, t.getTaskTrackerHttp(), t.getTaskId()); if (taskAttemptLogUrl != null) { ti.getLogUrls().add(taskAttemptLogUrl);
private static void downloadProfile(TaskCompletionEvent e ) throws IOException { URLConnection connection = new URL(getTaskLogURL(e.getTaskAttemptId(), e.getTaskTrackerHttp()) + "&filter=profile").openConnection(); InputStream in = connection.getInputStream(); OutputStream out = new FileOutputStream(e.getTaskAttemptId() + ".profile"); IOUtils.copyBytes(in, out, 64 * 1024, true); }
private static void downloadProfile(TaskCompletionEvent e ) throws IOException { URLConnection connection = new URL(getTaskLogURL(e.getTaskAttemptId(), e.getTaskTrackerHttp()) + "&filter=profile").openConnection(); InputStream in = connection.getInputStream(); OutputStream out = new FileOutputStream(e.getTaskAttemptId() + ".profile"); IOUtils.copyBytes(in, out, 64 * 1024, true); }
/** * List the events for the given job * @param jobId the job id for the job's events to list * @throws IOException */ private void listEvents(JobID jobId, int fromEventId, int numEvents) throws IOException { TaskCompletionEvent[] events = jobSubmitClient.getTaskCompletionEvents(jobId, fromEventId, numEvents); System.out.println("Task completion events for " + jobId); System.out.println("Number of events (from " + fromEventId + ") are: " + events.length); for(TaskCompletionEvent event: events) { System.out.println(event.getTaskStatus() + " " + event.getTaskAttemptId() + " " + getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp())); } }
/** * List the events for the given job * @param jobId the job id for the job's events to list * @throws IOException */ private void listEvents(JobID jobId, int fromEventId, int numEvents) throws IOException { TaskCompletionEvent[] events = jobSubmitClient.getTaskCompletionEvents(jobId, fromEventId, numEvents); System.out.println("Task completion events for " + jobId); System.out.println("Number of events (from " + fromEventId + ") are: " + events.length); for(TaskCompletionEvent event: events) { System.out.println(event.getTaskStatus() + " " + event.getTaskAttemptId() + " " + getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp())); } }
for (int i = 0; i < num_queries; i++) { TaskAttemptID taskAttemptID = failures.get(i).getTaskAttemptId(); String taskAttemptHost = failures.get(i).getTaskTrackerHttp(); String[] fails = job.getTaskDiagnostics(taskAttemptID); for (String failure : fails) {
@Override public boolean equals(Object o) { if(o == null) return false; if(o.getClass().equals(TaskCompletionEvent.class)) { TaskCompletionEvent event = (TaskCompletionEvent) o; return this.isMap == event.isMapTask() && this.eventId == event.getEventId() && this.idWithinJob == event.idWithinJob() && this.status.equals(event.getTaskStatus()) && this.taskId.equals(event.getTaskAttemptId()) && this.taskRunTime == event.getTaskRunTime() && this.taskTrackerHttp.equals(event.getTaskTrackerHttp()); } return false; }
@Override public boolean equals(Object o) { if(o == null) return false; if(o.getClass().equals(TaskCompletionEvent.class)) { TaskCompletionEvent event = (TaskCompletionEvent) o; return this.isMap == event.isMapTask() && this.eventId == event.getEventId() && this.idWithinJob == event.idWithinJob() && this.status.equals(event.getTaskStatus()) && this.taskId.equals(event.getTaskAttemptId()) && this.taskRunTime == event.getTaskRunTime() && this.taskTrackerHttp.equals(event.getTaskTrackerHttp()); } return false; }
ti.getLogUrls().add(getTaskAttemptLogUrl(t.getTaskTrackerHttp(), t.getTaskId()));
@Override public void resolve(TaskCompletionEvent event) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(reduceId, event.getTaskTrackerHttp()); addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); maxMapRuntime = Math.max(maxMapRuntime, event.getTaskRunTime()); break; case FAILED: case KILLED: case OBSOLETE: obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } }
@Override public void resolve(TaskCompletionEvent event) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(reduceId, event.getTaskTrackerHttp()); addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); maxMapRuntime = Math.max(maxMapRuntime, event.getTaskRunTime()); break; case FAILED: case KILLED: case OBSOLETE: obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } }
@Override public void resolve(TaskCompletionEvent event) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(reduceId, event.getTaskTrackerHttp()); addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); maxMapRuntime = Math.max(maxMapRuntime, event.getTaskRunTime()); break; case FAILED: case KILLED: case OBSOLETE: obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } }
mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(), mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE, mapEvent.getTaskTrackerHttp()));
@Override public void resolve(TaskCompletionEvent event) { switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(reduceId, event.getTaskTrackerHttp()); addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(), event.getTaskAttemptId()); maxMapRuntime = Math.max(maxMapRuntime, event.getTaskRunTime()); break; case FAILED: case KILLED: case OBSOLETE: obsoleteMapOutput(event.getTaskAttemptId()); LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + " map-task: '" + event.getTaskAttemptId() + "'"); break; case TIPFAILED: tipFailed(event.getTaskAttemptId().getTaskID()); LOG.info("Ignoring output of failed map TIP: '" + event.getTaskAttemptId() + "'"); break; } }
conf, t.getTaskTrackerHttp(), t.getTaskId()); if (taskAttemptLogUrl != null) { ti.getLogUrls().add(taskAttemptLogUrl);
TaskCompletionEvent.Status.SUCCEEDED){ LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); case ALL: LOG.info(event.toString()); displayTaskLogs(event.getTaskAttemptId(), event.getTaskTrackerHttp()); break;
switch (event.getTaskStatus()) { case SUCCEEDED: URI u = getBaseURI(event.getTaskTrackerHttp()); scheduler.addKnownMapOutput(u.getHost() + ":" + u.getPort(), u.toString(),