@Override public void setContainerId(ContainerId containerId) { // In some cases, for example, global scheduling. It is possible that // container created without container-id assigned, so we will publish // container creation event to timeline service when id assigned. container.setId(containerId); if (containerId != null) { rmContext.getRMApplicationHistoryWriter().containerStarted(this); } } }
private void sendATSCreateEvent() { rmContext.getRMApplicationHistoryWriter().applicationStarted(this); rmContext.getSystemMetricsPublisher().appCreated(this, this.startTime); String appViewACLs = submissionContext.getAMContainerSpec() .getApplicationACLs().get(ApplicationAccessType.VIEW_APP); rmContext.getSystemMetricsPublisher().appACLsUpdated( this, appViewACLs, systemClock.getTime()); }
.DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO); rmContext.getRMApplicationHistoryWriter().containerStarted(this);
.DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO); rmContext.getRMApplicationHistoryWriter().containerStarted(this);
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey, ApplicationAttemptId appAttemptId, NodeId nodeId, String user, RMContext rmContext, long creationTime, String nodeLabelExpression, boolean isExternallyAllocated) { this.stateMachine = stateMachineFactory.make(this); this.nodeId = nodeId; this.container = container; this.allocatedSchedulerKey = schedulerKey; this.appAttemptId = appAttemptId; this.user = user; this.creationTime = creationTime; this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); this.isAMContainer = false; this.nodeLabelExpression = nodeLabelExpression; this.lastConfirmedResource = container.getResource(); this.isExternallyAllocated = isExternallyAllocated; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); this.writeLock = lock.writeLock(); saveNonAMContainerMetaInfo = shouldPublishNonAMContainerEventstoATS(rmContext); if (container.getId() != null) { rmContext.getRMApplicationHistoryWriter().containerStarted(this); } if (this.container != null) { this.allocationTags = this.container.getAllocationTags(); } }
rmContext.getRMApplicationHistoryWriter().applicationStarted(this); rmContext.getSystemMetricsPublisher().appCreated(this, startTime);
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) throws IOException { Dispatcher dispatcher = mock(Dispatcher.class); when(rmContext.getDispatcher()).thenReturn(dispatcher); EventHandler eventHandler = mock(EventHandler.class); when(dispatcher.getEventHandler()).thenReturn(eventHandler); QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class); queInfo.setQueueName("testqueue"); when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean())) .thenReturn(queInfo); when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean())) .thenThrow(new IOException("queue does not exist")); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps); when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn( getSchedulerApps(apps)); ResourceScheduler rs = mock(ResourceScheduler.class); when(rmContext.getScheduler()).thenReturn(rs); }
@Override public void transition(RMContainerImpl container, RMContainerEvent event) { RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event; container.finishTime = System.currentTimeMillis(); container.finishedStatus = finishedEvent.getRemoteContainerStatus(); // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container updateAttemptMetrics(container); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus(), container.getAllocatedNode())); container.rmContext.getRMApplicationHistoryWriter().containerFinished( container); boolean saveNonAMContainerMetaInfo = container.rmContext.getYarnConfiguration().getBoolean( YarnConfiguration .APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO, YarnConfiguration .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO); if (saveNonAMContainerMetaInfo || container.isAMContainer()) { container.rmContext.getSystemMetricsPublisher().containerFinished( container, container.finishTime); } }
@Override public void transition(RMContainerImpl container, RMContainerEvent event) { RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event; container.finishTime = System.currentTimeMillis(); container.finishedStatus = finishedEvent.getRemoteContainerStatus(); // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container updateAttemptMetrics(container); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus(), container.getAllocatedNode())); container.rmContext.getRMApplicationHistoryWriter().containerFinished( container); boolean saveNonAMContainerMetaInfo = container.rmContext.getYarnConfiguration().getBoolean( YarnConfiguration .APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO, YarnConfiguration .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO); if (saveNonAMContainerMetaInfo || container.isAMContainer()) { container.rmContext.getSystemMetricsPublisher().containerFinished( container, container.finishTime); } }
@Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime; ClusterMetrics.getMetrics().addAMRegisterDelay(delay); RMAppAttemptRegistrationEvent registrationEvent = (RMAppAttemptRegistrationEvent) event; appAttempt.host = registrationEvent.getHost(); appAttempt.rpcPort = registrationEvent.getRpcport(); appAttempt.originalTrackingUrl = sanitizeTrackingUrl(registrationEvent.getTrackingurl()); // Let the app know appAttempt.eventHandler.handle(new RMAppEvent(appAttempt .getAppAttemptId().getApplicationId(), RMAppEventType.ATTEMPT_REGISTERED)); // TODO:FIXME: Note for future. Unfortunately we only do a state-store // write at AM launch time, so we don't save the AM's tracking URL anywhere // as that would mean an extra state-store write. For now, we hope that in // work-preserving restart, AMs are forced to reregister. appAttempt.rmContext.getRMApplicationHistoryWriter() .applicationAttemptStarted(appAttempt); appAttempt.rmContext.getSystemMetricsPublisher() .appAttemptRegistered(appAttempt, System.currentTimeMillis()); } }
@Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime; ClusterMetrics.getMetrics().addAMRegisterDelay(delay); RMAppAttemptRegistrationEvent registrationEvent = (RMAppAttemptRegistrationEvent) event; appAttempt.host = registrationEvent.getHost(); appAttempt.rpcPort = registrationEvent.getRpcport(); appAttempt.originalTrackingUrl = sanitizeTrackingUrl(registrationEvent.getTrackingurl()); // Let the app know appAttempt.eventHandler.handle(new RMAppEvent(appAttempt .getAppAttemptId().getApplicationId(), RMAppEventType.ATTEMPT_REGISTERED)); // TODO:FIXME: Note for future. Unfortunately we only do a state-store // write at AM launch time, so we don't save the AM's tracking URL anywhere // as that would mean an extra state-store write. For now, we hope that in // work-preserving restart, AMs are forced to reregister. appAttempt.rmContext.getRMApplicationHistoryWriter() .applicationAttemptStarted(appAttempt); appAttempt.rmContext.getSystemMetricsPublisher() .appAttemptRegistered(appAttempt, System.currentTimeMillis()); } }
anyBoolean())).thenThrow(new IOException("queue does not exist")); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
@Override public void transition(RMContainerImpl container, RMContainerEvent event) { RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event; container.finishTime = System.currentTimeMillis(); container.finishedStatus = finishedEvent.getRemoteContainerStatus(); // Inform AppAttempt // container.getContainer() can return null when a RMContainer is a // reserved container updateAttemptMetrics(container); container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus(), container.getAllocatedNode())); container.rmContext.getRMApplicationHistoryWriter().containerFinished( container); boolean saveNonAMContainerMetaInfo = shouldPublishNonAMContainerEventstoATS(container.rmContext); if (saveNonAMContainerMetaInfo || container.isAMContainer()) { container.rmContext.getSystemMetricsPublisher().containerFinished( container, container.finishTime); } }
appAttempt.rmContext.getRMApplicationHistoryWriter() .applicationAttemptStarted(appAttempt); appAttempt.rmContext.getSystemMetricsPublisher()
public void transition(RMAppImpl app, RMAppEvent event) { for (NodeId nodeId : app.getRanNodes()) { app.handler.handle( new RMNodeCleanAppEvent(nodeId, app.applicationId)); } app.finishTime = app.storedFinishTime; if (app.finishTime == 0 ) { app.finishTime = app.systemClock.getTime(); } // Recovered apps that are completed were not added to scheduler, so no // need to remove them from scheduler. if (app.recoveredFinalState == null) { app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, finalState)); } app.handler.handle( new RMAppManagerEvent(app.applicationId, RMAppManagerEventType.APP_COMPLETED)); app.rmContext.getRMApplicationHistoryWriter() .applicationFinished(app, finalState); app.rmContext.getSystemMetricsPublisher() .appFinished(app, finalState, app.finishTime); }; }
public void transition(RMAppImpl app, RMAppEvent event) { for (NodeId nodeId : app.getRanNodes()) { app.handler.handle( new RMNodeCleanAppEvent(nodeId, app.applicationId)); } app.finishTime = app.storedFinishTime; if (app.finishTime == 0 ) { app.finishTime = app.systemClock.getTime(); } // Recovered apps that are completed were not added to scheduler, so no // need to remove them from scheduler. if (app.recoveredFinalState == null) { app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, finalState)); } app.handler.handle( new RMAppManagerEvent(app.applicationId, RMAppManagerEventType.APP_COMPLETED)); app.rmContext.getRMApplicationHistoryWriter() .applicationFinished(app, finalState); app.rmContext.getSystemMetricsPublisher() .appFinished(app, finalState, app.finishTime); }; }
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
public void transition(RMAppImpl app, RMAppEvent event) { app.logAggregationStartTime = app.systemClock.getTime(); for (NodeId nodeId : app.getRanNodes()) { app.handler.handle( new RMNodeCleanAppEvent(nodeId, app.applicationId)); } app.finishTime = app.storedFinishTime; if (app.finishTime == 0 ) { app.finishTime = app.systemClock.getTime(); } // Recovered apps that are completed were not added to scheduler, so no // need to remove them from scheduler. if (app.recoveredFinalState == null) { app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, finalState)); } app.handler.handle( new RMAppManagerEvent(app.applicationId, RMAppManagerEventType.APP_COMPLETED)); app.rmContext.getRMApplicationHistoryWriter() .applicationFinished(app, finalState); app.rmContext.getSystemMetricsPublisher() .appFinished(app, finalState, app.finishTime); // set the memory free app.clearUnusedFields(); appAdminClientCleanUp(app); }; }
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());