private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
@Override public int getTaskAttemptNumber() { return taskAttemptID.getId(); }
@Override public int getIdentifier() { return attemptId.getId(); }
@Override public void transition(TaskImpl task, TaskEvent event) { TezTaskAttemptID successTaId = ((TaskEventTAUpdate)event).getTaskAttemptID(); task.taskAttemptStatus.put(successTaId.getId(), true); } }
public static TaskAttemptID toMRTaskAttemptId( TezTaskAttemptID taskAttemptId) { return new TaskAttemptID( toMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); }
this.taskAttemptStatus.put(attempt.getID().getId(), false);
@Override public void transition(TaskImpl task, TaskEvent event) { TaskEventTAUpdate castEvent = (TaskEventTAUpdate) event; task.addDiagnosticInfo("TaskAttempt " + castEvent.getTaskAttemptID().getId() + " killed"); if (task.commitAttempt !=null && castEvent.getTaskAttemptID().equals(task.commitAttempt)) { task.commitAttempt = null; } task.handleTaskAttemptCompletion( castEvent.getTaskAttemptID(), TaskAttemptStateInternal.KILLED); // we KillWaitAttemptCompletedTransitionready have a spare task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true); task.getVertex().incrementKilledTaskAttemptCount(); if (task.shouldScheduleNewAttempt()) { task.addAndScheduleAttempt(castEvent.getTaskAttemptID()); } } }
TezEvent tezEvent = eventIterator.next(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); if (taskIndex == taskId.getId()) {
@Override public void onStateChanged(TaskImpl task, TaskStateInternal taskStateInternal) { // Only registered for SUCCEEDED notifications at the moment Preconditions.checkState(taskStateInternal == TaskStateInternal.SUCCEEDED); TaskAttempt successfulAttempt = task.getSuccessfulAttempt(); // TODO TEZ-1577. // This is a horrible hack to get around recovery issues. Without this, recovery would fail // for successful vertices. // With this, recovery will end up failing for DAGs making use of InputInitializerEvents int succesfulAttemptInt = -1; if (successfulAttempt != null) { succesfulAttemptInt = successfulAttempt.getID().getId(); } task.stateChangeNotifier.taskSucceeded(task.getVertex().getName(), task.getTaskId(), succesfulAttemptInt); } }
@Override public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TaskEventTAUpdate castEvent = (TaskEventTAUpdate)event; task.handleTaskAttemptCompletion( castEvent.getTaskAttemptID(), TaskAttemptStateInternal.KILLED); task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true); // check whether all attempts are finished if (task.getFinishedAttemptsCount() == task.attempts.size()) { task.logJobHistoryTaskFailedEvent(getExternalState(TaskStateInternal.KILLED)); task.eventHandler.handle( new VertexEventTaskCompleted( task.taskId, getExternalState(TaskStateInternal.KILLED))); return TaskStateInternal.KILLED; } return task.getInternalState(); } }
String srcVertexName = tezEvent.getSourceInfo().getTaskVertexName(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); Task task = srcVertex.getTask(taskIndex); if (task.getState() == TaskState.SUCCEEDED) { successfulAttemptInteger = task.getSuccessfulAttempt().getID().getId(); vertexSuccessfulAttemptMap.put(taskIndex, successfulAttemptInteger);
public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getTaskID().getVertexID(); ApplicationId appId = vId.getDAGId().getApplicationId(); return new org.apache.hadoop.mapred.TaskAttemptID( new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp()) + String.valueOf(vId.getId()), appId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()), tezTaId.getId()); }
TaskEventTAFailed castEvent = (TaskEventTAFailed) event; schedulingCausalTA = castEvent.getTaskAttemptID(); task.addDiagnosticInfo("TaskAttempt " + castEvent.getTaskAttemptID().getId() + " failed," + " info=" + task.getAttempt(castEvent.getTaskAttemptID()).getDiagnostics()); if (task.commitAttempt != null && task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true); if (task.failedAttempts < task.maxFailedAttempts && castEvent.getTaskFailureType() == TaskFailureType.NON_FATAL) {
InputReadErrorEvent readErrorEvent = (InputReadErrorEvent)tezEvent.getEvent(); int failedInputIndexOnDestTa = readErrorEvent.getIndex(); if (readErrorEvent.getVersion() != attempt.getID().getId()) { throw new TezUncheckedException(attempt.getID() + " incorrectly blamed for read error from " + failedDestTaId
task.taskAttemptStatus.put(successTaId.getId(), true); task.successfulAttempt = successTaId; task.eventHandler.handle(new VertexEventTaskCompleted(
SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString());
((DataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId()); } else if (tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) { ((CompositeDataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId()); } else { ((InputFailedEvent) tezEvent.getEvent()).setVersion(srcTaId.getId());
task.taskAttemptStatus.put(failedAttemptId.getId(), true); return TaskStateInternal.SUCCEEDED;