@GET @Path("/jobs") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public JobsInfo getJobs(@Context HttpServletRequest hsr) { init(); JobsInfo allJobs = new JobsInfo(); for (Job job : appCtx.getAllJobs().values()) { // getAllJobs only gives you a partial we want a full Job fullJob = appCtx.getJob(job.getID()); if (fullJob == null) { continue; } allJobs.add(new JobInfo(fullJob, hasAccess(fullJob, hsr))); } return allJobs; }
public RMCommunicator(ClientService clientService, AppContext context) { super("RMCommunicator"); this.clientService = clientService; this.context = context; this.eventHandler = context.getEventHandler(); this.applicationId = context.getApplicationID(); this.stopped = new AtomicBoolean(false); this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>(); this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY); }
public AppInfo(App app, AppContext context) { this.appId = context.getApplicationID().toString(); this.name = context.getApplicationName().toString(); this.user = context.getUser().toString(); this.startedOn = context.getStartTime(); this.elapsedTime = Times.elapsed(this.startedOn, 0); }
protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), context.getClock(), conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)); addService(taskHeartbeatHandler); }
private static AppContext createAppContext() { ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); Job job = mock(Job.class); @SuppressWarnings("unchecked") EventHandler<Event> eventHandler = mock(EventHandler.class); AppContext ctx = mock(AppContext.class); when(ctx.getApplicationID()).thenReturn(appId); when(ctx.getApplicationAttemptId()).thenReturn(attemptId); when(ctx.getJob(isA(JobId.class))).thenReturn(job); when(ctx.getClusterInfo()).thenReturn( new ClusterInfo(Resource.newInstance(10240, 1))); when(ctx.getEventHandler()).thenReturn(eventHandler); return ctx; } }
private static CommitterEventHandler createCommitterEventHandler( Dispatcher dispatcher, OutputCommitter committer) { final SystemClock clock = SystemClock.getInstance(); AppContext appContext = mock(AppContext.class); when(appContext.getEventHandler()).thenReturn( dispatcher.getEventHandler()); when(appContext.getClock()).thenReturn(clock); RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() { @Override public long getLastHeartbeatTime() { return clock.getTime(); } @Override public void runOnNextHeartbeat(Runnable callback) { callback.run(); } }; ApplicationAttemptId id = ApplicationAttemptId.fromString( "appattempt_1234567890000_0001_0"); when(appContext.getApplicationID()).thenReturn(id.getApplicationId()); when(appContext.getApplicationAttemptId()).thenReturn(id); CommitterEventHandler handler = new CommitterEventHandler(appContext, committer, heartbeatHandler); dispatcher.register(CommitterEventType.class, handler); return handler; }
@SuppressWarnings("unchecked") @Override public void handle(JobEvent event) { ((EventHandler<JobEvent>)context.getJob(event.getJobId())).handle(event); } }
@Override public void contextualize(Configuration conf, AppContext context) { this.context = context; Map<JobId, Job> allJobs = context.getAllJobs(); for (Map.Entry<JobId, Job> entry : allJobs.entrySet()) { final Job job = entry.getValue(); mapperStatistics.put(job, new DataStatistics()); reducerStatistics.put(job, new DataStatistics()); slowTaskRelativeTresholds.put (job, conf.getFloat(MRJobConfig.SPECULATIVE_SLOWTASK_THRESHOLD,1.0f)); } }
private AppContext mockAppContext(Class<? extends AppContext> contextClass, ApplicationId appId, boolean isLastAMRetry) { JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(appId)); AppContext mockContext = mock(contextClass); Job mockJob = mockJob(); when(mockContext.getJob(jobId)).thenReturn(mockJob); when(mockContext.getApplicationID()).thenReturn(appId); when(mockContext.isLastAMRetry()).thenReturn(isLastAMRetry); if (mockContext instanceof RunningAppContext) { when(((RunningAppContext)mockContext).getTimelineClient()). thenReturn(TimelineClient.createTimelineClient()); when(((RunningAppContext) mockContext).getTimelineV2Client()) .thenReturn(TimelineV2Client .createTimelineClient(ApplicationId.newInstance(0, 1))); } return mockContext; }
taskAttemptFinishingMonitor.start(); when(appCtx.getEventHandler()).thenReturn(eventHandler); when(appCtx.getNMHostname()).thenReturn("0.0.0.0"); when(appCtx.getTaskAttemptFinishingMonitor()).thenReturn( taskAttemptFinishingMonitor); when(appCtx.getClock()).thenReturn(clock); org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP); TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0); appCtx.getTaskAttemptFinishingMonitor().register(attemptId); int check = 0; while ( !eventHandler.timedOut && check++ < 10 ) {
/** * MAPREDUCE-6771. Test if RMContainerAllocator generates the events in the * right order while processing finished containers. */ @Test public void testHandlingFinishedContainers() { EventHandler eventHandler = mock(EventHandler.class); AppContext context = mock(RunningAppContext.class); when(context.getClock()).thenReturn(new ControlledClock()); when(context.getClusterInfo()).thenReturn( new ClusterInfo(Resource.newInstance(10240, 1))); when(context.getEventHandler()).thenReturn(eventHandler); RMContainerAllocator containerAllocator = new RMContainerAllocatorForFinishedContainer(null, context, mock(AMPreemptionPolicy.class)); ContainerStatus finishedContainer = ContainerStatus.newInstance( mock(ContainerId.class), ContainerState.COMPLETE, "", 0); containerAllocator.processFinishedContainer(finishedContainer); InOrder inOrder = inOrder(eventHandler); inOrder.verify(eventHandler).handle( isA(TaskAttemptDiagnosticsUpdateEvent.class)); inOrder.verify(eventHandler).handle(isA(TaskAttemptEvent.class)); inOrder.verifyNoMoreInteractions(); }
public LocalContainerAllocator(ClientService clientService, AppContext context, String nmHost, int nmPort, int nmHttpPort , ContainerId cId) { super(clientService, context); this.eventHandler = context.getEventHandler(); this.nmHost = nmHost; this.nmPort = nmPort; this.nmHttpPort = nmHttpPort; this.containerId = cId; }
public DefaultSpeculator(Configuration conf, AppContext context) { this(conf, context, context.getClock()); }
ContainerId cId = ContainerId.newContainerId(getContext() .getApplicationAttemptId(), containerCount++); container.setContainerToken(null); container.setNodeHttpAddress("localhost:8042"); getContext().getEventHandler() .handle( new TaskAttemptContainerAssignedEvent(event
private MapTaskImpl getMockMapTask(long clusterTimestamp, EventHandler eh) { ApplicationId appId = ApplicationId.newInstance(clusterTimestamp, 1); JobId jobId = MRBuilderUtils.newJobId(appId, 1); int partitions = 2; Path remoteJobConfFile = mock(Path.class); JobConf conf = new JobConf(); TaskAttemptListener taskAttemptListener = mock(TaskAttemptListener.class); Token<JobTokenIdentifier> jobToken = (Token<JobTokenIdentifier>) mock(Token.class); Credentials credentials = null; Clock clock = SystemClock.getInstance(); int appAttemptId = 3; MRAppMetrics metrics = mock(MRAppMetrics.class); Resource minContainerRequirements = mock(Resource.class); when(minContainerRequirements.getMemorySize()).thenReturn(1000L); ClusterInfo clusterInfo = mock(ClusterInfo.class); AppContext appContext = mock(AppContext.class); when(appContext.getClusterInfo()).thenReturn(clusterInfo); TaskSplitMetaInfo taskSplitMetaInfo = mock(TaskSplitMetaInfo.class); MapTaskImpl mapTask = new MapTaskImpl(jobId, partitions, eh, remoteJobConfFile, conf, taskSplitMetaInfo, taskAttemptListener, jobToken, credentials, clock, appAttemptId, metrics, appContext); return mapTask; }
conf, appContext.getClientToAMTokenSecretManager(), conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(), server.getListenerAddress().getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress);
@Override protected void register() { ApplicationAttemptId attemptId = getContext().getApplicationAttemptId(); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); try { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ugi.addTokenIdentifier(token.decodeIdentifier()); } catch (IOException e) { throw new YarnRuntimeException(e); } super.register(); }
public BlacklistedNodesInfo(AppContext appContext) { blacklistedNodes = appContext.getBlacklistedNodes(); }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
protected void startRpcServer() { Configuration conf = getConfig(); try { server = new RPC.Builder(conf).setProtocol(TaskUmbilicalProtocol.class) .setInstance(this).setBindAddress("0.0.0.0") .setPortRangeConfig(MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE) .setNumHandlers( conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)) .setVerbose(false).setSecretManager(jobTokenSecretManager).build(); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new MRAMPolicyProvider()); } server.start(); this.address = NetUtils.createSocketAddrForHost( context.getNMHostname(), server.getListenerAddress().getPort()); } catch (IOException e) { throw new YarnRuntimeException(e); } }