@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
@Override public GetTaskReportsResponse getTaskReports( GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, JobACL.VIEW_JOB, true); Collection<Task> tasks = job.getTasks(taskType).values(); LOG.info("Getting task report for " + taskType + " " + jobId + ". Report-size will be " + tasks.size()); // Take lock to allow only one call, otherwise heap will blow up because // of counters in the report when there are multiple callers. synchronized (getTaskReportsLock) { for (Task task : tasks) { response.addTaskReport(task.getReport()); } } return response; }
@Override protected Collection<TaskAttempt> getTaskAttempts() { List<TaskAttempt> fewTaskAttemps = new ArrayList<TaskAttempt>(); String taskTypeStr = $(TASK_TYPE); TaskType taskType = MRApps.taskType(taskTypeStr); String attemptStateStr = $(ATTEMPT_STATE); TaskAttemptStateUI neededState = MRApps .taskAttemptState(attemptStateStr); for (Task task : super.app.getJob().getTasks(taskType).values()) { Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts(); for (TaskAttempt attempt : attempts.values()) { if (neededState.correspondsTo(attempt.getState())) { fewTaskAttemps.add(attempt); } } } return fewTaskAttemps; } }
public static Map<String, String> getTaskParams(AppContext appContext) { JobId jobId = appContext.getAllJobs().entrySet().iterator().next().getKey(); Entry<TaskId, Task> e = appContext.getJob(jobId).getTasks().entrySet().iterator().next(); e.getValue().getType(); Map<String, String> params = new HashMap<String, String>(); params.put(AMParams.JOB_ID, MRApps.toString(jobId)); params.put(AMParams.TASK_ID, MRApps.toString(e.getKey())); params.put(AMParams.TASK_TYPE, MRApps.taskSymbol(e.getValue().getType())); return params; }
@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
private void testTaskAttemptAssignedKilledHistory (FailingAttemptsDuringAssignedMRApp app) throws Exception { Configuration conf = new Configuration(); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Map<TaskId, Task> tasks = job.getTasks(); Task task = tasks.values().iterator().next(); app.waitForState(task, TaskState.SCHEDULED); Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts(); TaskAttempt attempt = attempts.values().iterator().next(); app.waitForState(attempt, TaskAttemptState.KILLED); waitFor(app::getTaStartJHEvent, 100, 800); waitFor(app::getTaKilledJHEvent, 100, 800); }
public void verifyAMTaskXML(NodeList nodes, Job job) { assertEquals("incorrect number of elements", 2, nodes.getLength()); for (Task task : job.getTasks().values()) { TaskId id = task.getID(); String tid = MRApps.toString(id); Boolean found = false; for (int i = 0; i < nodes.getLength(); i++) { Element element = (Element) nodes.item(i); if (tid.matches(WebServicesTestUtils.getXmlString(element, "id"))) { found = true; verifyAMSingleTaskXML(element, task); } } assertTrue("task with id: " + tid + " not in web service output", found); } }
@Test public void testTaskView() { AppContext appContext = new MockAppContext(0, 1, 1, 1); Map<String, String> params = getTaskParams(appContext); App app = new App(appContext); app.setJob(appContext.getAllJobs().values().iterator().next()); app.setTask(app.getJob().getTasks().values().iterator().next()); WebAppTests.testPage(TaskPage.class, App.class, app, params); }
@Test public void testSingleCounterView() { AppContext appContext = new MockAppContext(0, 1, 1, 1); Job job = appContext.getAllJobs().values().iterator().next(); // add a failed task to the job without any counters Task failedTask = MockJobs.newTask(job.getID(), 2, 1, true); Map<TaskId,Task> tasks = job.getTasks(); tasks.put(failedTask.getID(), failedTask); Map<String, String> params = getJobParams(appContext); params.put(AMParams.COUNTER_GROUP, "org.apache.hadoop.mapreduce.FileSystemCounter"); params.put(AMParams.COUNTER_NAME, "HDFS_WRITE_OPS"); WebAppTests.testPage(SingleCounterPage.class, AppContext.class, appContext, params); }
@Test public void testJobError() throws Exception { MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true); Job job = app.submit(new Configuration()); app.waitForState(job, JobState.RUNNING); Assert.assertEquals("Num tasks not correct", 1, job.getTasks().size()); Iterator<Task> it = job.getTasks().values().iterator(); Task task = it.next(); app.waitForState(task, TaskState.RUNNING); //send an invalid event on task at current state app.getContext().getEventHandler().handle( new TaskEvent( task.getID(), TaskEventType.T_SCHEDULE)); //this must lead to job error app.waitForState(job, JobState.ERROR); }
@Test public void testJobRebootNotLastRetryOnUnregistrationFailure() throws Exception { MRApp app = new MRApp(1, 0, false, this.getClass().getName(), true); Job job = app.submit(new Configuration()); app.waitForState(job, JobState.RUNNING); Assert.assertEquals("Num tasks not correct", 1, job.getTasks().size()); Iterator<Task> it = job.getTasks().values().iterator(); Task task = it.next(); app.waitForState(task, TaskState.RUNNING); //send an reboot event app.getContext().getEventHandler().handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT)); // return exteranl state as RUNNING since otherwise the JobClient will // prematurely exit. app.waitForState(job, JobState.RUNNING); }
private void testTaskAttemptAssignedFailHistory (FailingAttemptsDuringAssignedMRApp app) throws Exception { Configuration conf = new Configuration(); Job job = app.submit(conf); app.waitForState(job, JobState.FAILED); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertTrue("No Ta Started JH Event", app.getTaStartJHEvent()); Assert.assertTrue("No Ta Failed JH Event", app.getTaFailedJHEvent()); }
public void verifyAMTask(JSONArray arr, Job job, String type) throws JSONException { for (Task task : job.getTasks().values()) { TaskId id = task.getID(); String tid = MRApps.toString(id); Boolean found = false; if (type != null && task.getType() == MRApps.taskType(type)) { for (int i = 0; i < arr.length(); i++) { JSONObject info = arr.getJSONObject(i); if (tid.matches(info.getString("id"))) { found = true; verifyAMSingleTask(info, task); } } assertTrue("task with id: " + tid + " not in web service output", found); } } }
@Test public void testTaskAttemptsDefault() throws JSONException, Exception { WebResource r = resource(); Map<JobId, Job> jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); for (Task task : jobsMap.get(id).getTasks().values()) { String tid = MRApps.toString(task.getID()); ClientResponse response = r.path("ws").path("v1").path("mapreduce") .path("jobs").path(jobId).path("tasks").path(tid).path("attempts") .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyAMTaskAttempts(json, task); } } }
@Test public void testTaskAttemptsSlash() throws JSONException, Exception { WebResource r = resource(); Map<JobId, Job> jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); for (Task task : jobsMap.get(id).getTasks().values()) { String tid = MRApps.toString(task.getID()); ClientResponse response = r.path("ws").path("v1").path("mapreduce") .path("jobs").path(jobId).path("tasks").path(tid).path("attempts/") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyAMTaskAttempts(json, task); } } }
@Test public void testTaskAttempts() throws JSONException, Exception { WebResource r = resource(); Map<JobId, Job> jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); for (Task task : jobsMap.get(id).getTasks().values()) { String tid = MRApps.toString(task.getID()); ClientResponse response = r.path("ws").path("v1").path("mapreduce") .path("jobs").path(jobId).path("tasks").path(tid).path("attempts") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); verifyAMTaskAttempts(json, task); } } }
@Test public void testTaskIdSlash() throws JSONException, Exception { WebResource r = resource(); Map<JobId, Job> jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); for (Task task : jobsMap.get(id).getTasks().values()) { String tid = MRApps.toString(task.getID()); ClientResponse response = r.path("ws").path("v1").path("mapreduce") .path("jobs").path(jobId).path("tasks").path(tid + "/") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("task"); verifyAMSingleTask(info, task); } } }
@Test public void testTaskIdCountersSlash() throws JSONException, Exception { WebResource r = resource(); Map<JobId, Job> jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); for (Task task : jobsMap.get(id).getTasks().values()) { String tid = MRApps.toString(task.getID()); ClientResponse response = r.path("ws").path("v1").path("mapreduce") .path("jobs").path(jobId).path("tasks").path(tid).path("counters/") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("jobTaskCounters"); verifyAMJobTaskCounters(info, task); } } }