public RMCommunicator(ClientService clientService, AppContext context) { super("RMCommunicator"); this.clientService = clientService; this.context = context; this.eventHandler = context.getEventHandler(); this.applicationId = context.getApplicationID(); this.stopped = new AtomicBoolean(false); this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>(); this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY); }
public RMCommunicator(ClientService clientService, AppContext context) { super("RMCommunicator"); this.clientService = clientService; this.context = context; this.eventHandler = context.getEventHandler(); this.applicationId = context.getApplicationID(); this.stopped = new AtomicBoolean(false); this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>(); this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY); }
public RMCommunicator(ClientService clientService, AppContext context) { super("RMCommunicator"); this.clientService = clientService; this.context = context; this.eventHandler = context.getEventHandler(); this.applicationId = context.getApplicationID(); this.stopped = new AtomicBoolean(false); this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>(); this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY); }
public AppInfo(App app, AppContext context) { this.appId = context.getApplicationID().toString(); this.name = context.getApplicationName().toString(); this.user = context.getUser().toString(); this.startedOn = context.getStartTime(); this.elapsedTime = Times.elapsed(this.startedOn, 0); }
public AppInfo(App app, AppContext context) { this.appId = context.getApplicationID().toString(); this.name = context.getApplicationName().toString(); this.user = context.getUser().toString(); this.startedOn = context.getStartTime(); this.elapsedTime = Times.elapsed(this.startedOn, 0); }
public AppInfo(App app, AppContext context) { this.appId = context.getApplicationID().toString(); this.name = context.getApplicationName().toString(); this.user = context.getUser().toString(); this.startedOn = context.getStartTime(); this.elapsedTime = Times.elapsed(this.startedOn, 0); }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); commitThreadCancelTimeoutMs = conf.getInt( MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS); commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS, MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS); try { fs = FileSystem.get(conf); JobID id = TypeConverter.fromYarn(context.getApplicationID()); JobId jobId = TypeConverter.toYarn(id); String user = UserGroupInformation.getCurrentUser().getShortUserName(); startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId); endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId); endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId); } catch (IOException e) { throw new YarnRuntimeException(e); } }
@Override protected ContainerAllocator createContainerAllocator( ClientService clientService, AppContext context) { context = spy(context); when(context.getEventHandler()).thenReturn(null); when(context.getApplicationID()).thenReturn(null); return new CustomContainerAllocator(this, context); }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); commitThreadCancelTimeoutMs = conf.getInt( MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS); commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS, MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS); try { fs = FileSystem.get(conf); JobID id = TypeConverter.fromYarn(context.getApplicationID()); JobId jobId = TypeConverter.toYarn(id); String user = UserGroupInformation.getCurrentUser().getShortUserName(); startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId); endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId); endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId); } catch (IOException e) { throw new YarnRuntimeException(e); } }
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); commitThreadCancelTimeoutMs = conf.getInt( MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS); commitWindowMs = conf.getLong(MRJobConfig.MR_AM_COMMIT_WINDOW_MS, MRJobConfig.DEFAULT_MR_AM_COMMIT_WINDOW_MS); try { fs = FileSystem.get(conf); JobID id = TypeConverter.fromYarn(context.getApplicationID()); JobId jobId = TypeConverter.toYarn(id); String user = UserGroupInformation.getCurrentUser().getShortUserName(); startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId); endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId); endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId); } catch (IOException e) { throw new YarnRuntimeException(e); } }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
private static AppContext createAppContext() { ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); Job job = mock(Job.class); @SuppressWarnings("unchecked") EventHandler<Event> eventHandler = mock(EventHandler.class); AppContext ctx = mock(AppContext.class); when(ctx.getApplicationID()).thenReturn(appId); when(ctx.getApplicationAttemptId()).thenReturn(attemptId); when(ctx.getJob(isA(JobId.class))).thenReturn(job); when(ctx.getClusterInfo()).thenReturn( new ClusterInfo(Resource.newInstance(10240, 1))); when(ctx.getEventHandler()).thenReturn(eventHandler); return ctx; } }
@Before public void setUp() throws IOException { AppContext context = mock(AppContext.class); when(context.getApplicationID()).thenReturn( ApplicationId.newInstance(0, 0)); when(context.getApplicationName()).thenReturn("AppName"); when(context.getUser()).thenReturn("User"); when(context.getStartTime()).thenReturn(System.currentTimeMillis()); job = mock(Job.class); Task task = mock(Task.class); when(job.getTask(any(TaskId.class))).thenReturn(task); when(job.loadConfFile()).thenReturn(new Configuration()); when(job.getConfFile()).thenReturn(new Path("/")); JobId jobID = MRApps.toJobID("job_01_01"); when(context.getJob(jobID)).thenReturn(job); when(job.checkAccess(any(UserGroupInformation.class), any(JobACL.class))) .thenReturn(true); App app = new App(context); Configuration configuration = new Configuration(); ctx = mock(RequestContext.class); appController = new AppControllerForTest(app, configuration, ctx); appController.getProperty().put(AMParams.JOB_ID, "job_01_01"); appController.getProperty().put(AMParams.TASK_ID, taskId); }
private AppContext mockAppContext(Class<? extends AppContext> contextClass, ApplicationId appId, boolean isLastAMRetry) { JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(appId)); AppContext mockContext = mock(contextClass); Job mockJob = mockJob(); when(mockContext.getJob(jobId)).thenReturn(mockJob); when(mockContext.getApplicationID()).thenReturn(appId); when(mockContext.isLastAMRetry()).thenReturn(isLastAMRetry); if (mockContext instanceof RunningAppContext) { when(((RunningAppContext)mockContext).getTimelineClient()). thenReturn(TimelineClient.createTimelineClient()); when(((RunningAppContext) mockContext).getTimelineV2Client()) .thenReturn(TimelineV2Client .createTimelineClient(ApplicationId.newInstance(0, 1))); } return mockContext; }
private static CommitterEventHandler createCommitterEventHandler( Dispatcher dispatcher, OutputCommitter committer) { final SystemClock clock = SystemClock.getInstance(); AppContext appContext = mock(AppContext.class); when(appContext.getEventHandler()).thenReturn( dispatcher.getEventHandler()); when(appContext.getClock()).thenReturn(clock); RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() { @Override public long getLastHeartbeatTime() { return clock.getTime(); } @Override public void runOnNextHeartbeat(Runnable callback) { callback.run(); } }; ApplicationAttemptId id = ApplicationAttemptId.fromString( "appattempt_1234567890000_0001_0"); when(appContext.getApplicationID()).thenReturn(id.getApplicationId()); when(appContext.getApplicationAttemptId()).thenReturn(id); CommitterEventHandler handler = new CommitterEventHandler(appContext, committer, heartbeatHandler); dispatcher.register(CommitterEventType.class, handler); return handler; }
@Test public void testAppControllerIndex() { AppContext ctx = new MockAppContext(0, 1, 1, 1); Injector injector = WebAppTests.createMockInjector(AppContext.class, ctx); AppController controller = injector.getInstance(AppController.class); controller.index(); assertEquals(ctx.getApplicationID().toString(), controller.get(APP_ID,"")); }
public void verifyAMInfoGeneric(AppContext ctx, String id, String user, String name, long startedOn, long elapsedTime) { WebServicesTestUtils.checkStringMatch("id", ctx.getApplicationID() .toString(), id); WebServicesTestUtils.checkStringMatch("user", ctx.getUser().toString(), user); WebServicesTestUtils.checkStringMatch("name", ctx.getApplicationName(), name); assertEquals("startedOn incorrect", ctx.getStartTime(), startedOn); assertTrue("elapsedTime not greater then 0", (elapsedTime > 0)); }
@Test (timeout=50000) public void testSetTrackingURLAfterHistoryIsWritten() throws Exception { TestParams t = new TestParams(true); Configuration conf = new Configuration(); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0, false); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); verify(jheh, times(0)).processDoneFiles(any(JobId.class)); verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); // Job finishes and successfully writes history handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme( conf, t.mockAppContext.getApplicationID()); verify(t.mockAppContext, times(1)).setHistoryUrl(historyUrl); } finally { jheh.stop(); } }