private Resource getResourceInfoFromContainerRequest( TaskAttemptImpl taImpl, EventHandler eventHandler) { taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_SCHEDULE)); assertEquals("Task attempt is not in STARTING state", taImpl.getState(), TaskAttemptState.STARTING); ArgumentCaptor<Event> captor = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(2)).handle(captor.capture()); List<ContainerRequestEvent> containerRequestEvents = new ArrayList<>(); for (Event e : captor.getAllValues()) { if (e instanceof ContainerRequestEvent) { containerRequestEvents.add((ContainerRequestEvent) e); } } assertEquals("Expected one ContainerRequestEvent after scheduling " + "task attempt", 1, containerRequestEvents.size()); return containerRequestEvents.get(0).getCapability(); }
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { // Status update calls don't really change the state of the attempt. TaskAttemptStatus newReportedStatus = ((TaskAttemptStatusUpdateEvent) event) .getReportedTaskAttemptStatus(); // Now switch the information in the reportedStatus taskAttempt.reportedStatus = newReportedStatus; taskAttempt.reportedStatus.taskState = taskAttempt.getState(); // send event to speculator about the reported status taskAttempt.eventHandler.handle (new SpeculatorEvent (taskAttempt.reportedStatus, taskAttempt.clock.getTime())); taskAttempt.updateProgressSplits(); //if fetch failures are present, send the fetch failure event to job //this only will happen in reduce attempt type if (taskAttempt.reportedStatus.fetchFailedMaps != null && taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) { taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent( taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps)); } } }
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { TaskAttemptStatusUpdateEvent statusEvent = ((TaskAttemptStatusUpdateEvent)event); AtomicReference<TaskAttemptStatus> taskAttemptStatusRef = statusEvent.getTaskAttemptStatusRef(); TaskAttemptStatus newReportedStatus = taskAttemptStatusRef.getAndSet(null); // Now switch the information in the reportedStatus taskAttempt.reportedStatus = newReportedStatus; taskAttempt.reportedStatus.taskState = taskAttempt.getState(); // send event to speculator about the reported status taskAttempt.eventHandler.handle (new SpeculatorEvent (taskAttempt.reportedStatus, taskAttempt.clock.getTime())); taskAttempt.updateProgressSplits(); //if fetch failures are present, send the fetch failure event to job //this only will happen in reduce attempt type if (taskAttempt.reportedStatus.fetchFailedMaps != null && taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) { String hostname = taskAttempt.container == null ? "UNKNOWN" : taskAttempt.container.getNodeId().getHost(); taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent( taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps, hostname)); } } }
@SuppressWarnings("unchecked") @Override public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { // Status update calls don't really change the state of the attempt. TaskAttemptStatus newReportedStatus = ((TaskAttemptStatusUpdateEvent) event) .getReportedTaskAttemptStatus(); // Now switch the information in the reportedStatus taskAttempt.reportedStatus = newReportedStatus; taskAttempt.reportedStatus.taskState = taskAttempt.getState(); // send event to speculator about the reported status taskAttempt.eventHandler.handle (new SpeculatorEvent (taskAttempt.reportedStatus, taskAttempt.clock.getTime())); taskAttempt.updateProgressSplits(); //if fetch failures are present, send the fetch failure event to job //this only will happen in reduce attempt type if (taskAttempt.reportedStatus.fetchFailedMaps != null && taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) { String hostname = taskAttempt.container == null ? "UNKNOWN" : taskAttempt.container.getNodeId().getHost(); taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent( taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps, hostname)); } } }
mock(Map.class))); taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); assertEquals("Task attempt is not in running state", taImpl.getState(), TaskAttemptState.RUNNING); taImpl.handle(new TaskAttemptEvent(attemptId,
container, mock(Map.class))); taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); assertEquals("Task attempt is not in running state", taImpl.getState(), TaskAttemptState.RUNNING); taImpl.handle(new TaskAttemptEvent(attemptId,
TaskAttemptEventType.TA_CONTAINER_COMPLETED)); assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId, reduceTAId, "Host")); assertEquals("Task attempt is not in FAILED state", taImpl.getState(), TaskAttemptState.FAILED); taImpl.handle(new TaskAttemptEvent(attemptId, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)); assertEquals("Task attempt is not in FAILED state, still", taImpl.getState(), TaskAttemptState.FAILED); assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
TaskAttemptEventType.TA_COMMIT_PENDING)); assertEquals("Task attempt is not in commit pending state", taImpl.getState(), TaskAttemptState.COMMIT_PENDING); taImpl.handle(new TaskAttemptEvent(attemptId,
mock(Map.class))); taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); assertEquals("Task attempt is not in running state", taImpl.getState(), TaskAttemptState.RUNNING); taImpl.handle(new TaskAttemptEvent(attemptId,
@Override public TaskAttemptReport getReport() { TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class); readLock.lock(); try { result.setTaskAttemptId(attemptId); //take the LOCAL state of attempt //DO NOT take from reportedStatus result.setTaskAttemptState(getState()); result.setProgress(reportedStatus.progress); result.setStartTime(launchTime); result.setFinishTime(finishTime); result.setShuffleFinishTime(this.reportedStatus.shuffleFinishTime); result.setDiagnosticInfo(StringUtils.join(LINE_SEPARATOR, getDiagnostics())); result.setPhase(reportedStatus.phase); result.setStateString(reportedStatus.stateString); result.setCounters(TypeConverter.toYarn(getCounters())); result.setContainerId(this.getAssignedContainerID()); result.setNodeManagerHost(trackerName); result.setNodeManagerHttpPort(httpPort); if (this.container != null) { result.setNodeManagerPort(this.container.getNodeId().getPort()); } return result; } finally { readLock.unlock(); } }
@Override public TaskAttemptReport getReport() { TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class); readLock.lock(); try { result.setTaskAttemptId(attemptId); //take the LOCAL state of attempt //DO NOT take from reportedStatus result.setTaskAttemptState(getState()); result.setProgress(reportedStatus.progress); result.setStartTime(launchTime); result.setFinishTime(finishTime); result.setShuffleFinishTime(this.reportedStatus.shuffleFinishTime); result.setDiagnosticInfo(StringUtils.join(LINE_SEPARATOR, getDiagnostics())); result.setPhase(reportedStatus.phase); result.setStateString(reportedStatus.stateString); result.setCounters(TypeConverter.toYarn(getCounters())); result.setContainerId(this.getAssignedContainerID()); result.setNodeManagerHost(trackerName); result.setNodeManagerHttpPort(httpPort); if (this.container != null) { result.setNodeManagerPort(this.container.getNodeId().getPort()); } return result; } finally { readLock.unlock(); } }
@Override public TaskAttemptReport getReport() { TaskAttemptReport result = recordFactory.newRecordInstance(TaskAttemptReport.class); readLock.lock(); try { result.setTaskAttemptId(attemptId); //take the LOCAL state of attempt //DO NOT take from reportedStatus result.setTaskAttemptState(getState()); result.setProgress(reportedStatus.progress); result.setStartTime(launchTime); result.setFinishTime(finishTime); result.setShuffleFinishTime(this.reportedStatus.shuffleFinishTime); result.setDiagnosticInfo(StringUtils.join(LINE_SEPARATOR, getDiagnostics())); result.setPhase(reportedStatus.phase); result.setStateString(reportedStatus.stateString); result.setCounters(TypeConverter.toYarn(getCounters())); result.setContainerId(this.getAssignedContainerID()); result.setNodeManagerHost(trackerName); result.setNodeManagerHttpPort(httpPort); if (this.container != null) { result.setNodeManagerPort(this.container.getNodeId().getPort()); } return result; } finally { readLock.unlock(); } }
private void containerKillBeforeAssignment(boolean scheduleAttempt) throws Exception { MockEventHandler eventHandler = new MockEventHandler(); ApplicationId appId = ApplicationId.newInstance(1, 2); JobId jobId = MRBuilderUtils.newJobId(appId, 1); TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, mock(Path.class), 1, mock(TaskSplitMetaInfo.class), new JobConf(), mock(TaskAttemptListener.class), mock(Token.class), new Credentials(), SystemClock.getInstance(), mock(AppContext.class)); if (scheduleAttempt) { taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_SCHEDULE)); } taImpl.handle(new TaskAttemptKillEvent(taImpl.getID(),"", true)); assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.KILLED); assertEquals("Task attempt's internal state is not KILLED", taImpl.getInternalState(), TaskAttemptStateInternal.KILLED); assertFalse("InternalError occurred", eventHandler.internalError); TaskEvent event = eventHandler.lastTaskEvent; assertEquals(TaskEventType.T_ATTEMPT_KILLED, event.getType()); // In NEW state, new map attempt should not be rescheduled. assertFalse(((TaskTAttemptKilledEvent)event).getRescheduleAttempt()); }
@Test public void testTaskAttemptDiagnosticEventOnFinishing() throws Exception { MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + "SUCCESS_FINISHING_CONTAINER", taImpl.getInternalState(), TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER); // TA_DIAGNOSTICS_UPDATE doesn't change state taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(taImpl.getID(), "Task got updated")); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + "SUCCESS_FINISHING_CONTAINER", taImpl.getInternalState(), TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER); assertFalse("InternalError occurred", eventHandler.internalError); }
@Test public void testTimeoutWhileSuccessFinishing() throws Exception { MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + "SUCCESS_FINISHING_CONTAINER", taImpl.getInternalState(), TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER); // If the task stays in SUCCESS_FINISHING_CONTAINER for too long, // TaskAttemptListenerImpl will time out the attempt. taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_TIMED_OUT)); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + "SUCCESS_CONTAINER_CLEANUP", taImpl.getInternalState(), TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP); assertFalse("InternalError occurred", eventHandler.internalError); }
TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in SUCCEEDED state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.KILLED); assertEquals("Task attempt's internal state is not KILL_CONTAINER_CLEANUP", TaskAttemptEventType.TA_CLEANUP_DONE)); assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.KILLED);
@Test public void testFailMapTaskByClient() throws Exception { MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_FAILMSG_BY_CLIENT)); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals("Task attempt's internal state is not " + "FAIL_CONTAINER_CLEANUP", taImpl.getInternalState(), TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_CONTAINER_CLEANED)); assertEquals("Task attempt's internal state is not FAIL_TASK_CLEANUP", taImpl.getInternalState(), TaskAttemptStateInternal.FAIL_TASK_CLEANUP); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_CLEANUP_DONE)); assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.FAILED); assertFalse("InternalError occurred", eventHandler.internalError); }
assertEquals("Task attempt is not in FAILED state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals("Task attempt's internal state is not " + assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals("Task attempt's internal state is not " + TaskAttemptEventType.TA_CLEANUP_DONE)); assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.FAILED);
@Test public void testKillMapTaskAfterSuccess() throws Exception { MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in SUCCEEDED state", taImpl.getState(), TaskAttemptState.SUCCEEDED); assertEquals("Task attempt's internal state is not " + "SUCCESS_FINISHING_CONTAINER", taImpl.getInternalState(), TaskAttemptStateInternal.SUCCESS_FINISHING_CONTAINER); taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_CONTAINER_CLEANED)); // Send a map task attempt kill event indicating next map attempt has to be // reschedule taImpl.handle(new TaskAttemptKillEvent(taImpl.getID(),"", true)); assertEquals("Task attempt is not in KILLED state", taImpl.getState(), TaskAttemptState.KILLED); assertEquals("Task attempt's internal state is not KILLED", taImpl.getInternalState(), TaskAttemptStateInternal.KILLED); assertFalse("InternalError occurred", eventHandler.internalError); TaskEvent event = eventHandler.lastTaskEvent; assertEquals(TaskEventType.T_ATTEMPT_KILLED, event.getType()); // Send an attempt killed event to TaskImpl forwarding the same reschedule // flag we received in task attempt kill event. assertTrue(((TaskTAttemptKilledEvent)event).getRescheduleAttempt()); }
@Test public void testTimeoutWhileFailFinishing() throws Exception { MockEventHandler eventHandler = new MockEventHandler(); TaskAttemptImpl taImpl = createTaskAttemptImpl(eventHandler); taImpl.handle(new TaskAttemptFailEvent(taImpl.getID())); assertEquals("Task attempt is not in RUNNING state", taImpl.getState(), TaskAttemptState.FAILED); assertEquals("Task attempt's internal state is not " + "FAIL_FINISHING_CONTAINER", taImpl.getInternalState(), TaskAttemptStateInternal.FAIL_FINISHING_CONTAINER); // If the task stays in FAIL_FINISHING_CONTAINER for too long, // TaskAttemptListenerImpl will time out the attempt. taImpl.handle(new TaskAttemptEvent(taImpl.getID(), TaskAttemptEventType.TA_TIMED_OUT)); assertEquals("Task attempt's internal state is not FAIL_CONTAINER_CLEANUP", taImpl.getInternalState(), TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP); assertFalse("InternalError occurred", eventHandler.internalError); }