@SuppressWarnings("resource") @Test public void testJobSuccess() throws Exception { MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true, false); JobImpl job = (JobImpl) app.submit(new Configuration()); app.waitForInternalState(job, JobStateInternal.SUCCEEDED); // AM is not unregistered Assert.assertEquals(JobState.RUNNING, job.getState()); // imitate that AM is unregistered app.successfullyUnregistered.set(true); app.waitForState(job, JobState.SUCCEEDED); }
@Test public void testAbsentNotificationOnNotLastRetryUnregistrationFailure() throws Exception { HttpServer2 server = startHttpServer(); MRApp app = spy(new MRAppWithCustomContainerAllocator(2, 2, false, this.getClass().getName(), true, 1, false)); doNothing().when(app).sysexit(); JobConf conf = new JobConf(); conf.set(JobContext.MR_JOB_END_NOTIFICATION_URL, JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.submit(conf); app.waitForState(job, JobState.RUNNING); app.getContext().getEventHandler() .handle(new JobEvent(app.getJobId(), JobEventType.JOB_AM_REBOOT)); app.waitForInternalState(job, JobStateInternal.REBOOT); // Now shutdown. // Unregistration fails: isLastAMRetry is recalculated, this is not app.shutDownJob(); // Not the last AM attempt. So user should that the job is still running. app.waitForState(job, JobState.RUNNING); Assert.assertFalse(app.isLastAMRetry()); Assert.assertEquals(0, JobEndServlet.calledTimes); Assert.assertNull(JobEndServlet.requestUri); Assert.assertNull(JobEndServlet.foundJobState); server.stop(); }
@Override protected void attemptLaunched(TaskAttemptId attemptID) { super.attemptLaunched(attemptID); //the task is launched and sends done immediately concurrentRunningTasks--; }
@Test public void testZeroMaps() throws Exception { MRApp app = new MRApp(0, 1, true, this.getClass().getName(), true); Job job = app.submit(new Configuration()); app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted(); }
@Test public void testZeroMapReduces() throws Exception{ MRApp app = new MRApp(0, 0, true, this.getClass().getName(), true); Job job = app.submit(new Configuration()); app.waitForState(job, JobState.SUCCEEDED); }
Job jobAttempt1 = app.submit(conf); app.waitForState(jobAttempt1, JobState.RUNNING); app.waitForState(mapper, TaskState.RUNNING); TaskAttempt mapAttempt = mapper.getAttempts().values().iterator().next(); app.waitForState(mapAttempt, TaskAttemptState.RUNNING); app.getContext().getEventHandler().handle( new TaskAttemptEvent(mapAttempt.getID(), TaskAttemptEventType.TA_DONE)); app.waitForState(mapper, TaskState.SUCCEEDED); app.stop(); Job jobAttempt2 = app.submit(conf); Assert.assertTrue("Recovery from previous job attempt is processed even " + "though intermediate data encryption is enabled.", !app.recovered()); app.waitForState(jobAttempt2, JobState.RUNNING); tasks = jobAttempt2.getTasks().values().iterator(); mapper = tasks.next(); app.waitForState(mapper, TaskState.RUNNING); mapAttempt = mapper.getAttempts().values().iterator().next(); app.waitForState(mapAttempt, TaskAttemptState.RUNNING); app.getContext().getEventHandler().handle( new TaskAttemptEvent(mapAttempt.getID(), TaskAttemptEventType.TA_DONE)); app.waitForState(mapper, TaskState.SUCCEEDED); app.waitForState(redAttempt, TaskAttemptState.RUNNING); app.getContext().getEventHandler().handle(
mrApp.submit(conf); Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next() .getValue(); DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher(); .getContainerAllocator(); mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING); mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values() .iterator().next(), TaskAttemptStateInternal.UNASSIGNED); mrApp.waitForState(t, TaskState.RUNNING);
doNothing().when(app).sysexit(); JobConf conf = new JobConf(); conf.set(JobContext.MR_JOB_END_NOTIFICATION_URL, JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.submit(conf); app.waitForState(job, JobState.RUNNING); app.getContext().getEventHandler() .handle(new JobEvent(app.getJobId(), JobEventType.JOB_AM_REBOOT)); app.waitForInternalState(job, JobStateInternal.REBOOT); app.waitForServiceToStop(10000); Assert.assertFalse(app.isLastAMRetry());
clock.setTime(10); MRApp app = new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock); app.setAllocatedContainerResource(containerResource); Configuration conf = new Configuration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minContainerSize); app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1))); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 2, tasks.size()); Iterator<Task> taskIter = tasks.values().iterator(); Task mTask = taskIter.next(); app.waitForState(mTask, TaskState.RUNNING); Task rTask = taskIter.next(); app.waitForState(rTask, TaskState.RUNNING); Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts(); Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size()); TaskAttempt mta = mAttempts.values().iterator().next(); TaskAttempt rta = rAttempts.values().iterator().next(); app.waitForState(mta, TaskAttemptState.RUNNING); app.waitForState(rta, TaskAttemptState.RUNNING); app.getContext() .getEventHandler() .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
@Test public void testPreviousJobOutputCleanedWhenNoRecovery() throws Exception { int runCount = 0; MRApp app = new MRAppWithHistory(1, 2, false, this.getClass().getName(), true, ++runCount); Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, false); conf.setClass("mapred.output.committer.class", TestFileOutputCommitter.class, org.apache.hadoop.mapred.OutputCommitter.class); conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); conf.set(FileOutputFormat.OUTDIR, outputDir.toString()); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Assert.assertEquals("No of tasks not correct", 3, job.getTasks().size()); //stop the app before the job completes. app.stop(); app.close(); //rerun app = new MRAppWithHistory(1, 2, false, this.getClass().getName(), false, ++runCount); job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Assert.assertEquals("No of tasks not correct", 3, job.getTasks().size()); TestFileOutputCommitter committer = ( TestFileOutputCommitter) app.getCommitter(); assertTrue("commiter.abortJob() has not been called", committer.isAbortJobCalled()); app.close(); }
.equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name() : Policy.HTTP_ONLY.name()); webProxyBase = "/proxy/" + app.getAppID(); conf.set("hadoop.http.filter.initializers", TestAMFilterInitializer.class.getName()); Job job = app.submit(conf); String hostPort = NetUtils.getHostPortString(((MRClientService) app.getClientService()) .getWebApp().getListenerAddress()); URL httpUrl = new URL("http://" + hostPort + "/mapreduce"); + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce", true); Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY, conn.getResponseCode()); app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted();
@Override protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) { UserGroupInformation currentUser = null; try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnRuntimeException(e); } Job newJob = new TestJob(getJobId(), getAttemptID(), conf, getDispatcher().getEventHandler(), getTaskAttemptListener(), getContext().getClock(), getCommitter(), isNewApiCommitter(), currentUser.getUserName(), getContext(), forcedState, diagnostic); ((AppContext) getContext()).getAllJobs().put(newJob.getID(), newJob); getDispatcher().register(JobFinishEvent.Type.class, new EventHandler<JobFinishEvent>() { @Override public void handle(JobFinishEvent event) { stop(); } }); return newJob; }
@Test public void testMapFailureMaxPercent() throws Exception { MRApp app = new MockFirstFailingTaskMRApp(4, 0); Configuration conf = new Configuration(); //reduce the no of attempts so test run faster conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); conf.setInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 1); conf.setInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 20); conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1); Job job = app.submit(conf); app.waitForState(job, JobState.FAILED); //setting the failure percentage to 25% (1/4 is 25) will //make the Job successful app = new MockFirstFailingTaskMRApp(4, 0); conf = new Configuration(); //reduce the no of attempts so test run faster conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 2); conf.setInt(MRJobConfig.REDUCE_MAX_ATTEMPTS, 1); conf.setInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 25); conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1); job = app.submit(conf); app.waitForState(job, JobState.SUCCEEDED); }
private void testNotificationOnLastRetry(boolean withRuntimeException) throws Exception { HttpServer2 server = startHttpServer(); // Act like it is the second attempt. Default max attempts is 2 MRApp app = spy(new MRAppWithCustomContainerAllocator( 2, 2, true, this.getClass().getName(), true, 2, true)); doNothing().when(app).sysexit(); JobConf conf = new JobConf(); conf.set(JobContext.MR_JOB_END_NOTIFICATION_URL, JobEndServlet.baseUrl + "jobend?jobid=$jobId&status=$jobStatus"); JobImpl job = (JobImpl)app.submit(conf); app.waitForInternalState(job, JobStateInternal.SUCCEEDED); // Unregistration succeeds: successfullyUnregistered is set if (withRuntimeException) { YarnRuntimeException runtimeException = new YarnRuntimeException( new ClosedChannelException()); doThrow(runtimeException).when(app).stop(); } app.shutDownJob(); Assert.assertTrue(app.isLastAMRetry()); Assert.assertEquals(1, JobEndServlet.calledTimes); Assert.assertEquals("jobid=" + job.getID() + "&status=SUCCEEDED", JobEndServlet.requestUri.getQuery()); Assert.assertEquals(JobState.SUCCEEDED.toString(), JobEndServlet.foundJobState); server.stop(); }
@Test public void testTaskFailWithUnusedContainer() throws Exception { MRApp app = new MRAppWithFailingTaskAndUnusedContainer(); Configuration conf = new Configuration(); int maxAttempts = 1; conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts); // disable uberization (requires entire job to be reattempted, so max for // subtask attempts is overridden to 1) conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); app.waitForState(task, TaskState.SCHEDULED); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator() .next().getAttempts(); Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts .size()); TaskAttempt attempt = attempts.values().iterator().next(); app.waitForInternalState((TaskAttemptImpl) attempt, TaskAttemptStateInternal.ASSIGNED); app.getDispatcher().getEventHandler().handle( new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_CONTAINER_COMPLETED)); app.waitForState(job, JobState.FAILED); }
Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); long am1StartTime = app.getAllAMInfos().get(0).getStartTime(); app.waitForState(mapTask, TaskState.RUNNING); TaskAttempt taskAttempt = mapTask.getAttempts().values().iterator().next(); app.waitForState(taskAttempt, TaskAttemptState.RUNNING); app.stop(); job = app.submit(conf); app.waitForState(job, JobState.RUNNING); Assert.assertEquals("No of tasks not correct", 1, job.getTasks().size()); it = job.getTasks().values().iterator(); mapTask = it.next(); List<AMInfo> amInfos = app.getAllAMInfos(); Assert.assertEquals(2, amInfos.size()); AMInfo amInfoOne = amInfos.get(0); Assert.assertEquals(am1StartTime, amInfoOne.getStartTime()); app.stop();
Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); app.waitForState(task, TaskState.SCHEDULED); app.waitForInternalState((TaskAttemptImpl) attempt, TaskAttemptStateInternal.ASSIGNED); app.waitForState(job, JobState.FAILED); app.stop();
MRApp app = new BlockingMRApp(2, 0, latch); Job job = app.submit(new Configuration()); app.waitForState(job, JobState.RUNNING); Map<TaskId,Task> tasks = job.getTasks(); Assert.assertEquals("No of tasks is not correct", 2, app.waitForState(task1, TaskState.SCHEDULED); app.waitForState(task2, TaskState.SCHEDULED); app.getContext().getEventHandler().handle( new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_KILL)); app.waitForState(job, JobState.SUCCEEDED);
@Test public void testContainerPassThrough() throws Exception { MRApp app = new MRApp(0, 1, true, this.getClass().getName(), true) { @Override protected ContainerLauncher createContainerLauncher(AppContext context) { return new MockContainerLauncher() { @Override public void handle(ContainerLauncherEvent event) { if (event instanceof ContainerRemoteLaunchEvent) { containerObtainedByContainerLauncher = ((ContainerRemoteLaunchEvent) event).getAllocatedContainer(); } super.handle(event); } }; }; }; Job job = app.submit(new Configuration()); app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted(); Collection<Task> tasks = job.getTasks().values(); Collection<TaskAttempt> taskAttempts = tasks.iterator().next().getAttempts().values(); TaskAttemptImpl taskAttempt = (TaskAttemptImpl) taskAttempts.iterator().next(); // Container from RM should pass through to the launcher. Container object // should be the same. Assert.assertTrue(taskAttempt.container == containerObtainedByContainerLauncher); }
private void finishTask(DrainDispatcher rmDispatcher, MockNM node, MRApp mrApp, Task task) throws Exception { TaskAttempt attempt = task.getAttempts().values().iterator().next(); List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1); contStatus.add(ContainerStatus.newInstance(attempt.getAssignedContainerID(), ContainerState.COMPLETE, "", 0)); Map<ApplicationId,List<ContainerStatus>> statusUpdate = new HashMap<ApplicationId,List<ContainerStatus>>(1); statusUpdate.put(mrApp.getAppID(), contStatus); node.nodeHeartbeat(statusUpdate, true); rmDispatcher.await(); mrApp.getContext().getEventHandler().handle( new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE)); mrApp.waitForState(task, TaskState.SUCCEEDED); }