if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) { notifier = new JobEndNotifier(); notifier.setConf(getConfig()); notifyIsLastAMRetry(isLastAMRetry); MRAppMaster.this.stop(); sendJobEndNotify(notifier); notifier = null; } catch (Throwable t) { LOG.warn("Graceful stop failed. Exiting.. ", t); exitMRAppMaster(1, t); } finally { if (isLastAMRetry && notifier != null) { sendJobEndNotify(notifier); exitMRAppMaster(0, null);
@Override public void run() { shutDownJob(); } }.start();
public void run() { LOG.info("MRAppMaster received a signal. Signaling RMCommunicator and " + "JobHistoryEventHandler."); // Notify the JHEH and RMCommunicator that a SIGTERM has been received so // that they don't take too long in shutting down if(appMaster.containerAllocator instanceof ContainerAllocatorRouter) { ((ContainerAllocatorRouter) appMaster.containerAllocator) .setSignalled(true); } appMaster.notifyIsLastAMRetry(appMaster.isLastAMRetry); appMaster.stop(); } }
createJobClassLoader(conf); initJobCredentialsAndUGI(conf); committer = createOutputCommitter(conf); try { String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); FileSystem fs = getFileSystem(conf); forcedState = JobStateInternal.FAILED; } else { if (isCommitJobRepeatable()) { cleanupInterruptedCommit(conf, fs, startCommitFile); } else { dispatcher = createDispatcher(); addIfService(dispatcher); if (copyHistory) { historyService = createJobHistoryHandler(context); dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, historyService); addService(createStagingDirCleaningService()); containerAllocator = createContainerAllocator(null, context);
public void testDeletionofStagingOnReboot() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); fs = mock(FileSystem.class); when(fs.delete(any(Path.class),anyBoolean())).thenReturn(true); String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, JobStateInternal.REBOOT, 1); //no retry appMaster.init(conf); appMaster.start(); //shutdown the job, is lastRetry appMaster.shutDownJob(); //test whether notifyIsLastAMRetry called Assert.assertEquals(true, ((TestMRApp)appMaster).getTestIsLastAMRetry()); verify(fs).delete(stagingJobPath, true); }
System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(containerIdStr, Environment.CONTAINER_ID.name()); validateInputParam(nodeHostString, Environment.NM_HOST.name()); validateInputParam(nodePortString, Environment.NM_PORT.name()); validateInputParam(nodeHttpPortString, Environment.NM_HTTP_PORT.name()); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); new MRAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), appSubmitTime); .getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); initAndStartAppMaster(appMaster, conf, jobUserName); } catch (Throwable t) { LOG.fatal("Error starting MRAppMaster", t);
boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { appMaster.stop();
notifyIsLastAMRetry(isLastAMRetry); MRAppMaster.this.stop(); if (getConfig().get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL) != null) { try { LOG.info("Job end notification started for jobID : " + job.getReport().getJobId()); JobEndNotifier notifier = new JobEndNotifier(); notifier.setConf(getConfig()); JobReport report = job.getReport();
@Test public void testMRAppMasterForDifferentUser() throws IOException, InterruptedException { String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001"; String containerIdStr = "container_1317529182569_0004_000001_1"; String userName = "TestAppMasterUser"; ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString( applicationAttemptIdStr); ContainerId containerId = ContainerId.fromString(containerIdStr); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, System.currentTimeMillis()); JobConf conf = new JobConf(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); Path userPath = new Path(stagingDir, userName); Path userStagingPath = new Path(userPath, ".staging"); assertEquals(userStagingPath.toString(), appMaster.stagingDirPath.toString()); }
public void testDeletionofStagingOnKillLastTry() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry appMaster.init(conf); assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry()); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook(appMaster); hook.run(); assertTrue("MRAppMaster isn't stopped", appMaster.isInState(Service.STATE.STOPPED)); verify(fs).delete(stagingJobPath, true); }
@Override public void notifyIsLastAMRetry(boolean isLastAMRetry){ testIsLastAMRetry = isLastAMRetry; super.notifyIsLastAMRetry(isLastAMRetry); }
@Override protected ContainerAllocator createContainerAllocator( final ClientService clientService, final AppContext context) { if(allocator == null) { if (crushUnregistration) { return new CustomContainerAllocator(context); } else { return super.createContainerAllocator(clientService, context); } } return allocator; }
@Override protected EventHandler<JobHistoryEvent> createJobHistoryHandler( AppContext context) { spyHistoryService = Mockito.spy((JobHistoryEventHandler) super .createJobHistoryHandler(context)); spyHistoryService.setForcejobCompletion(this.isLastAMRetry); return spyHistoryService; } }
@Test (timeout = 30000) public void testDeletionofStagingOnKill() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0); JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); appMaster.init(conf); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook(appMaster); hook.run(); verify(fs, times(0)).delete(stagingJobPath, true); }
createJobClassLoader(conf); initJobCredentialsAndUGI(conf); dispatcher = createDispatcher(); addIfService(dispatcher); taskAttemptFinishingMonitor = createTaskAttemptFinishingMonitor(dispatcher.getEventHandler()); addIfService(taskAttemptFinishingMonitor); context = new RunningAppContext(conf, taskAttemptFinishingMonitor); committer = createOutputCommitter(conf); try { String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); FileSystem fs = getFileSystem(conf); forcedState = JobStateInternal.FAILED; } else { if (isCommitJobRepeatable()) { cleanupInterruptedCommit(conf, fs, startCommitFile); } else { if (copyHistory) { historyService = createJobHistoryHandler(context); dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, historyService);
@Test (timeout = 30000) public void testNoDeletionofStagingOnReboot() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); fs = mock(FileSystem.class); when(fs.delete(any(Path.class),anyBoolean())).thenReturn(true); String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingDir = MRApps.getStagingAreaDir(conf, user); when(fs.exists(stagingDir)).thenReturn(true); ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1); MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); appMaster.init(conf); appMaster.start(); //shutdown the job, not the lastRetry appMaster.shutDownJob(); //test whether notifyIsLastAMRetry called Assert.assertEquals(false, ((TestMRApp)appMaster).getTestIsLastAMRetry()); verify(fs, times(0)).delete(stagingJobPath, true); }
System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); validateInputParam(containerIdStr, Environment.CONTAINER_ID.name()); validateInputParam(nodeHostString, Environment.NM_HOST.name()); validateInputParam(nodePortString, Environment.NM_PORT.name()); validateInputParam(nodeHttpPortString, Environment.NM_HTTP_PORT.name()); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); new MRAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), appSubmitTime); .getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); initAndStartAppMaster(appMaster, conf, jobUserName); } catch (Throwable t) { LOG.error("Error starting MRAppMaster", t);
boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); } catch (IOException e) { assertTrue(appMaster.errorHappenedShutDown); assertEquals(JobStateInternal.SUCCEEDED, appMaster.forcedState); appMaster.stop();
public void run() { LOG.info("MRAppMaster received a signal. Signaling RMCommunicator and " + "JobHistoryEventHandler."); // Notify the JHEH and RMCommunicator that a SIGTERM has been received so // that they don't take too long in shutting down if(appMaster.containerAllocator instanceof ContainerAllocatorRouter) { ((ContainerAllocatorRouter) appMaster.containerAllocator) .setSignalled(true); } appMaster.notifyIsLastAMRetry(appMaster.isLastAMRetry); appMaster.stop(); } }