public static TaskCompletionEvent fromYarn( TaskAttemptCompletionEvent newEvent) { return new TaskCompletionEvent(newEvent.getEventId(), fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(), newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP), fromYarn(newEvent.getStatus()), newEvent.getMapOutputServerAddress()); }
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); int mapEventIdx = -1; if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) { TaskAttemptId attemptId = tce.getAttemptId(); TaskId taskId = attemptId.getTaskId(); TaskAttemptCompletionEvent successEvent = job.taskAttemptCompletionEvents.get(successEventNo); successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE); int mapCompletionIdx = job.taskCompletionIdxToMapCompletionIdx.get(successEventNo); if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) { job.successAttemptCompletionEventNoMap.put(taskId, tce.getEventId());
private static TaskAttemptCompletionEvent createTce(int eventId, boolean isMap, TaskAttemptCompletionEventStatus status) { JobId jid = MRBuilderUtils.newJobId(12345, 1, 1); TaskId tid = MRBuilderUtils.newTaskId(jid, 0, isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE); TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0); RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(eventId); tce.setAttemptId(attemptId); tce.setStatus(status); return tce; }
1, events.length); Assert.assertEquals("Event status not correct", TaskAttemptCompletionEventStatus.SUCCEEDED, events[0].getStatus()); TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); 4, events.length); Assert.assertEquals("Event map attempt id not correct", mapAttempt1.getID(), events[0].getAttemptId()); Assert.assertEquals("Event map attempt id not correct", mapAttempt1.getID(), events[1].getAttemptId()); Assert.assertEquals("Event map attempt id not correct", mapAttempt2.getID(), events[2].getAttemptId()); Assert.assertEquals("Event redude attempt id not correct", reduceAttempt.getID(), events[3].getAttemptId()); Assert.assertEquals("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); Assert.assertEquals("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus.FAILED, events[1].getStatus()); Assert.assertEquals("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus()); Assert.assertEquals("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());
Records.newRecord(TaskAttemptCompletionEvent.class); TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); tce.setAttemptId(attemptId); tce.setStatus(TaskAttemptCompletionEventStatus.SUCCEEDED); job.handle(new JobTaskAttemptCompletedEvent(tce));
public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) { super(completionEvent.getAttemptId().getTaskId().getJobId(), JobEventType.JOB_TASK_ATTEMPT_COMPLETED); this.completionEvent = completionEvent; }
1, events.length); Assert.assertEquals("Event status not correct", TaskAttemptCompletionEventStatus.SUCCEEDED, events[0].getStatus());
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); int mapEventIdx = -1; if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) { TaskAttemptId attemptId = tce.getAttemptId(); TaskId taskId = attemptId.getTaskId(); TaskAttemptCompletionEvent successEvent = job.taskAttemptCompletionEvents.get(successEventNo); successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE); int mapCompletionIdx = job.taskCompletionIdxToMapCompletionIdx.get(successEventNo); if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) { job.successAttemptCompletionEventNoMap.put(taskId, tce.getEventId());
1, events.length); Assert.assertEquals("Event status not correct", TaskAttemptCompletionEventStatus.SUCCEEDED, events[0].getStatus()); TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); 6, events.length); Assert.assertEquals("Event map attempt id not correct", mapAttempt1.getID(), events[0].getAttemptId()); Assert.assertEquals("Event map attempt id not correct", mapAttempt1.getID(), events[1].getAttemptId()); Assert.assertEquals("Event map attempt id not correct", mapAttempt2.getID(), events[2].getAttemptId()); Assert.assertEquals("Event reduce attempt id not correct", reduceAttempt.getID(), events[3].getAttemptId()); Assert.assertEquals("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus.OBSOLETE, events[0].getStatus()); Assert.assertEquals("Event status not correct for map attempt1", TaskAttemptCompletionEventStatus.FAILED, events[1].getStatus()); Assert.assertEquals("Event status not correct for map attempt2", TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus()); Assert.assertEquals("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());
public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) { super(completionEvent.getAttemptId().getTaskId().getJobId(), JobEventType.JOB_TASK_ATTEMPT_COMPLETED); this.completionEvent = completionEvent; }
private void handleTaskAttemptCompletion(TaskAttemptId attemptId, TaskAttemptCompletionEventStatus status) { TaskAttempt attempt = attempts.get(attemptId); //raise the completion event only if the container is assigned // to nextAttemptNumber if (attempt.getNodeHttpAddress() != null) { TaskAttemptCompletionEvent tce = recordFactory .newRecordInstance(TaskAttemptCompletionEvent.class); tce.setEventId(-1); String scheme = (encryptedShuffle) ? "https://" : "http://"; tce.setMapOutputServerAddress(StringInterner.weakIntern(scheme + attempt.getNodeHttpAddress().split(":")[0] + ":" + attempt.getShufflePort())); tce.setStatus(status); tce.setAttemptId(attempt.getID()); int runTime = 0; if (attempt.getFinishTime() != 0 && attempt.getLaunchTime() !=0) runTime = (int)(attempt.getFinishTime() - attempt.getLaunchTime()); tce.setAttemptRunTime(runTime); //raise the event to job so that it adds the completion event to its //data structures eventHandler.handle(new JobTaskAttemptCompletedEvent(tce)); } }
tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); int mapEventIdx = -1; if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) { TaskAttemptId attemptId = tce.getAttemptId(); TaskId taskId = attemptId.getTaskId(); TaskAttemptCompletionEvent successEvent = job.taskAttemptCompletionEvents.get(successEventNo); successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE); int mapCompletionIdx = job.taskCompletionIdxToMapCompletionIdx.get(successEventNo); if (TaskAttemptCompletionEventStatus.SUCCEEDED.equals(tce.getStatus())) { job.successAttemptCompletionEventNoMap.put(taskId, tce.getEventId());
public static TaskCompletionEvent fromYarn( TaskAttemptCompletionEvent newEvent) { return new TaskCompletionEvent(newEvent.getEventId(), fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(), newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP), fromYarn(newEvent.getStatus()), newEvent.getMapOutputServerAddress()); }
public JobTaskAttemptCompletedEvent(TaskAttemptCompletionEvent completionEvent) { super(completionEvent.getAttemptId().getTaskId().getJobId(), JobEventType.JOB_TASK_ATTEMPT_COMPLETED); this.completionEvent = completionEvent; }
tace.setAttemptId(taskAttempt.getID()); tace.setAttemptRunTime(attemptRunTime); tace.setEventId(eventId++); tace.setMapOutputServerAddress(taskAttempt .getAssignedContainerMgrAddress()); tace.setStatus(taceStatus); completionEvents.add(tace); if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP) {
public static TaskCompletionEvent fromYarn( TaskAttemptCompletionEvent newEvent) { return new TaskCompletionEvent(newEvent.getEventId(), fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(), newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP), fromYarn(newEvent.getStatus()), newEvent.getMapOutputServerAddress()); }
tace.setAttemptId(taskAttempt.getID()); tace.setAttemptRunTime(attemptRunTime); tace.setEventId(eventId++); tace.setMapOutputServerAddress(taskAttempt .getAssignedContainerMgrAddress()); tace.setStatus(taceStatus); completionEvents.add(tace); if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP) {
public static TaskCompletionEvent fromYarn( TaskAttemptCompletionEvent newEvent) { return new TaskCompletionEvent(newEvent.getEventId(), fromYarn(newEvent.getAttemptId()), newEvent.getAttemptId().getId(), newEvent.getAttemptId().getTaskId().getTaskType().equals(TaskType.MAP), fromYarn(newEvent.getStatus()), newEvent.getMapOutputServerAddress()); }