public DefaultSpeculator(Configuration conf, AppContext context) { this(conf, context, context.getClock()); }
public DefaultSpeculator(Configuration conf, AppContext context) { this(conf, context, context.getClock()); }
public DefaultSpeculator(Configuration conf, AppContext context) { this(conf, context, context.getClock()); }
public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); this.stopped = new AtomicBoolean(false); this.clock = context.getClock(); }
public RMContainerAllocator(ClientService clientService, AppContext context) { super(clientService, context); this.stopped = new AtomicBoolean(false); this.clock = context.getClock(); this.assignedRequests = createAssignedRequests(); }
public RMContainerAllocator(ClientService clientService, AppContext context, AMPreemptionPolicy preemptionPolicy) { super(clientService, context); this.preemptionPolicy = preemptionPolicy; this.stopped = new AtomicBoolean(false); this.clock = context.getClock(); this.assignedRequests = createAssignedRequests(); }
private synchronized void cancelJobCommit() { Thread threadCommitting = jobCommitThread; if (threadCommitting != null && threadCommitting.isAlive()) { LOG.info("Cancelling commit"); threadCommitting.interrupt(); // wait up to configured timeout for commit thread to finish long now = context.getClock().getTime(); long timeoutTimestamp = now + commitThreadCancelTimeoutMs; try { while (jobCommitThread == threadCommitting && now > timeoutTimestamp) { wait(now - timeoutTimestamp); now = context.getClock().getTime(); } } catch (InterruptedException e) { } } }
private synchronized void cancelJobCommit() { Thread threadCommitting = jobCommitThread; if (threadCommitting != null && threadCommitting.isAlive()) { LOG.info("Cancelling commit"); threadCommitting.interrupt(); // wait up to configured timeout for commit thread to finish long now = context.getClock().getTime(); long timeoutTimestamp = now + commitThreadCancelTimeoutMs; try { while (jobCommitThread == threadCommitting && now > timeoutTimestamp) { wait(now - timeoutTimestamp); now = context.getClock().getTime(); } } catch (InterruptedException e) { } } }
private synchronized void cancelJobCommit() { Thread threadCommitting = jobCommitThread; if (threadCommitting != null && threadCommitting.isAlive()) { LOG.info("Cancelling commit"); threadCommitting.interrupt(); // wait up to configured timeout for commit thread to finish long now = context.getClock().getTime(); long timeoutTimestamp = now + commitThreadCancelTimeoutMs; try { while (jobCommitThread == threadCommitting && now > timeoutTimestamp) { wait(now - timeoutTimestamp); now = context.getClock().getTime(); } } catch (InterruptedException e) { } } }
protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), context.getClock(), conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)); addService(taskHeartbeatHandler); }
protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), context.getClock(), conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)); addService(taskHeartbeatHandler); }
protected void registerHeartbeatHandler(Configuration conf) { taskHeartbeatHandler = new TaskHeartbeatHandler(context.getEventHandler(), context.getClock(), conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)); addService(taskHeartbeatHandler); }
@Override public long getLastHeartbeatTime() { return appContext.getClock().getTime(); } @Override
private synchronized void waitForValidCommitWindow() throws InterruptedException { long lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); long now = context.getClock().getTime(); while (now - lastHeartbeatTime > commitWindowMs) { rmHeartbeatHandler.runOnNextHeartbeat(new Runnable() { @Override public void run() { synchronized (EventProcessor.this) { EventProcessor.this.notify(); } } }); wait(); lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); now = context.getClock().getTime(); } } }
private synchronized void waitForValidCommitWindow() throws InterruptedException { long lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); long now = context.getClock().getTime(); while (now - lastHeartbeatTime > commitWindowMs) { rmHeartbeatHandler.runOnNextHeartbeat(new Runnable() { @Override public void run() { synchronized (EventProcessor.this) { EventProcessor.this.notify(); } } }); wait(); lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); now = context.getClock().getTime(); } } }
private static AppContext createAppContext( ApplicationAttemptId appAttemptId, Job job, Clock clock) { AppContext context = createAppContext(appAttemptId, job); when(context.getClock()).thenReturn(clock); return context; }
@Override public long getLastHeartbeatTime() { return getContext().getClock().getTime(); }
@Test(timeout = 2000) public void testRMContainerAllocatorExceptionIsHandled() throws Exception { ClientService mockClientService = mock(ClientService.class); AppContext mockContext = mock(AppContext.class); MockRMCommunicator mockRMCommunicator = new MockRMCommunicator(mockClientService, mockContext); RMCommunicator communicator = spy(mockRMCommunicator); Clock mockClock = mock(Clock.class); when(mockContext.getClock()).thenReturn(mockClock); doThrow(new RMContainerAllocationException("Test")).doNothing() .when(communicator).heartbeat(); when(mockClock.getTime()).thenReturn(1L).thenThrow(new AssertionError( "GetClock called second time, when it should not have since the " + "thread should have quit")); AllocatorRunnable testRunnable = communicator.new AllocatorRunnable(); testRunnable.run(); }
@Override protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) { UserGroupInformation currentUser = null; try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnRuntimeException(e); } Job newJob = new TestJob(getJobId(), getAttemptID(), conf, getDispatcher().getEventHandler(), getTaskAttemptListener(), getContext().getClock(), getCommitter(), isNewApiCommitter(), currentUser.getUserName(), getContext(), forcedState, diagnostic); ((AppContext) getContext()).getAllJobs().put(newJob.getID(), newJob); getDispatcher().register(JobFinishEvent.Type.class, createJobFinishEventHandler()); return newJob; }
@Override protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) { UserGroupInformation currentUser = null; try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnRuntimeException(e); } Job newJob = new TestJob(getJobId(), getAttemptID(), conf, getDispatcher() .getEventHandler(), getTaskAttemptListener(), getContext() .getClock(), getCommitter(), isNewApiCommitter(), currentUser.getUserName(), getContext(), forcedState, diagnostic); ((AppContext) getContext()).getAllJobs().put(newJob.getID(), newJob); getDispatcher().register(JobFinishEvent.Type.class, createJobFinishEventHandler()); return newJob; }