private void setLlapOfFragmentId(final ProcessorContext context) { // TODO: could we do this only if the OF is actually used? String attemptId = Converters.createTaskAttemptId(context).toString(); if (l4j.isDebugEnabled()) { l4j.debug("Setting the LLAP fragment ID for OF to " + attemptId); } jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); }
LOG.info( "Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}", taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd"); LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId); TerminateFragmentRequestProto.newBuilder().setQueryIdentifier( constructQueryIdentifierProto( taskAttemptId.getTaskID().getVertexID().getDAGId().getId())) .setFragmentIdentifierString(taskAttemptId.toString()).build(); communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() { LOG.info( "Not sending terminate request for fragment {} since it's node is not known. Already unregistered", taskAttemptId.toString());
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private SubmitWorkRequestProto constructSubmitWorkRequest(ContainerId containerId, TaskSpec taskSpec, FragmentRuntimeInfo fragmentRuntimeInfo, String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId()); builder.setContainerIdString(containerId.toString()); builder.setAmHost(getAmHostString()); builder.setAmPort(getAddress().getPort()); Preconditions.checkState(currentQueryIdentifierProto.getDagIndex() == taskSpec.getTaskAttemptID().getTaskID().getVertexID().getDAGId().getId()); builder.setCredentialsBinary( getCredentials(getContext().getCurrentDagInfo().getCredentials())); builder.setWorkSpec(VertexOrBinary.newBuilder().setVertex(Converters.constructSignableVertexSpec( taskSpec, currentQueryIdentifierProto, getTokenIdentifier(), user, hiveQueryId)).build()); // Don't call builder.setWorkSpecSignature() - Tez doesn't sign fragments builder.setFragmentRuntimeInfo(fragmentRuntimeInfo); if (scheduler != null) { // May be null in tests // TODO: see javadoc builder.setIsGuaranteed(scheduler.isInitialGuaranteed(taskSpec.getTaskAttemptID())); } return builder.build(); }
Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); if (LOG.isInfoEnabled()) { LOG.info("Queueing container for execution: fragemendId={}, {}", final String dagId = attemptId.getTaskID().getVertexID().getDAGId().toString(); final String queryId = vertex.getHiveQueryId(); final String fragmentId = LlapTezUtils.stripAttemptPrefix(fragmentIdString); env.put(ApplicationConstants.Environment.USER.name(), vertex.getUser()); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.fromString(fragmentIdString); int dagIdentifier = taskAttemptId.getTaskID().getVertexID().getDAGId().getId();
private String constructLlapLogUrl(final TezTaskAttemptID attemptID, final String containerIdString, final boolean isDone, final String nmAddress) { String dagId = attemptID.getTaskID().getVertexID().getDAGId().toString(); String filename = JOINER.join(currentHiveQueryId, "-", dagId, ".log", (isDone ? ".done" : ""), "?nm.id=", nmAddress); String url = PATH_JOINER.join(timelineServerUri, "ws", "v1", "applicationhistory", "containers", containerIdString, "logs", filename); return url; }
task.successfulAttempt = successTaId; LOG.info("Recovered a successful attempt" + ", taskAttemptId=" + task.successfulAttempt.toString()); !task.commitAttempt.equals(successTaId)) { task.taskAttemptStatus.put(successTaId.getId(), true); task.successfulAttempt = successTaId; task.eventHandler.handle(new VertexEventTaskCompleted( if (!attempt.getID().equals(task.successfulAttempt) &&
TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, getContext().getTaskVertexIndex()); TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber()); jobConf.set(MRInput.TEZ_MAPREDUCE_DAG_ID, tezDAGID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_VERTEX_ID, tezVertexID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ID, tezTaskID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ATTEMPT_ID, tezTaskAttemptID.toString());
TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo() .getTaskAttemptID(); int destTaskIndex = destAttemptId.getTaskID().getId(); int srcTaskIndex; int numConsumers; TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, taskAttemptIndex); sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId,
case Constants.TEZ_TASK_ATTEMPT_ID: String taskAttemptName = entity; TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName); if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) { LOG.warn(taskAttemptName + " does not belong to " + tezDAGID); continue;
@Override public int getTaskAttemptNumber() { return taskAttemptID.getId(); }
@Override public void transition(TaskImpl task, TaskEvent event) { TaskEventTAUpdate castEvent = (TaskEventTAUpdate) event; task.addDiagnosticInfo("TaskAttempt " + castEvent.getTaskAttemptID().getId() + " killed"); if (task.commitAttempt !=null && castEvent.getTaskAttemptID().equals(task.commitAttempt)) { task.commitAttempt = null; } task.handleTaskAttemptCompletion( castEvent.getTaskAttemptID(), TaskAttemptStateInternal.KILLED); // we KillWaitAttemptCompletedTransitionready have a spare task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true); task.getVertex().incrementKilledTaskAttemptCount(); if (task.shouldScheduleNewAttempt()) { task.addAndScheduleAttempt(castEvent.getTaskAttemptID()); } } }
private void updateHeartbeatInfo( String hostname, String uniqueId, int port, TezAttemptArray tasks) { int updateCount = 0; HashSet<TezTaskAttemptID> attempts = new HashSet<>(); for (Writable w : tasks.get()) { attempts.add((TezTaskAttemptID)w); } String error = ""; for (Map.Entry<String, LlapTaskUmbilicalExternalClient> entry : registeredClients.entrySet()) { LlapTaskUmbilicalExternalClient registeredClient = entry.getValue(); if (doesClientMatchHeartbeat(registeredClient, hostname, uniqueId, port)) { TezTaskAttemptID ta = TezTaskAttemptID.fromString(registeredClient.requestInfo.taskAttemptId); if (attempts.contains(ta)) { registeredClient.setLastHeartbeat(System.currentTimeMillis()); updateCount++; } else { error += (registeredClient.requestInfo.taskAttemptId + ", "); } } } if (!error.isEmpty()) { LOG.info("The tasks we expected to be on the node are not there: " + error); } if (updateCount == 0) { LOG.info("No tasks found for heartbeat from hostname " + hostname + ", port " + port); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if(o == null) { return false; } if (o.getClass() == this.getClass()) { TaskAttemptIdentifierImpl other = (TaskAttemptIdentifierImpl) o; return this.attemptId.equals(other.attemptId); } else { return false; } }
private static int vertexNum(TaskInfo taskInfo) { return taskInfo.getAttemptId().getTaskID().getVertexID().getId(); // Sigh... }
Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); if (LOG.isInfoEnabled()) { LOG.info("Queueing container for execution: fragemendId={}, {}", final String dagId = attemptId.getTaskID().getVertexID().getDAGId().toString(); final String queryId = vertex.getHiveQueryId(); final String fragmentId = LlapTezUtils.stripAttemptPrefix(fragmentIdString); env.put(ApplicationConstants.Environment.USER.name(), vertex.getUser()); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.fromString(fragmentIdString); int dagIdentifier = taskAttemptId.getTaskID().getVertexID().getDAGId().getId();
TezTaskAttemptID taskAttemptId = TezTaskAttemptID.fromString(attemptInfo .getTaskAttemptId()); VertexInfo vertexInfo = dagInfo.getVertexFromId(taskAttemptId.getTaskID() .getVertexID().toString()); Preconditions.checkState(vertexInfo != null, "Vertex " + taskAttemptId.getTaskID() .getVertexID().toString() + " is not present in DAG"); TaskInfo taskInfo = vertexInfo.getTask(taskAttemptId.getTaskID().toString()); attemptInfo.setTaskInfo(taskInfo);
@Override public int getIdentifier() { return attemptId.getId(); }
TaskEventTAFailed castEvent = (TaskEventTAFailed) event; schedulingCausalTA = castEvent.getTaskAttemptID(); task.addDiagnosticInfo("TaskAttempt " + castEvent.getTaskAttemptID().getId() + " failed," + " info=" + task.getAttempt(castEvent.getTaskAttemptID()).getDiagnostics()); if (task.commitAttempt != null && castEvent.getTaskAttemptID().equals(task.commitAttempt)) { task.commitAttempt = null; task.taskAttemptStatus.put(castEvent.getTaskAttemptID().getId(), true); if (task.failedAttempts < task.maxFailedAttempts && castEvent.getTaskFailureType() == TaskFailureType.NON_FATAL) {