private void setLlapOfFragmentId(final ProcessorContext context) { // TODO: could we do this only if the OF is actually used? String attemptId = Converters.createTaskAttemptId(context).toString(); if (l4j.isDebugEnabled()) { l4j.debug("Setting the LLAP fragment ID for OF to " + attemptId); } jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); }
private void setLlapOfFragmentId(final ProcessorContext context) { // TODO: could we do this only if the OF is actually used? String attemptId = Converters.createTaskAttemptId(context).toString(); if (l4j.isDebugEnabled()) { l4j.debug("Setting the LLAP fragment ID for OF to " + attemptId); } jconf.set(LlapOutputFormat.LLAP_OF_ID_KEY, attemptId); }
@Override public void taskKilled(TezTaskAttemptID taskAttemptId) throws IOException { String taskAttemptIdString = taskAttemptId.toString(); LlapTaskUmbilicalExternalClient client = registeredClients.get(taskAttemptIdString); if (client != null) { if (client.requestInfo.state == RequestState.PENDING) { // A task kill while the request is still in PENDING state means the request should be retried. LOG.info("Received task kill for {} which is still in pending state. Retry submission.", taskAttemptIdString); client.retrySubmission(); } else { try { LOG.error("Task killed - " + taskAttemptIdString); client.unregisterClient(); if (client.responder != null) { client.responder.taskKilled(taskAttemptId); } } catch (Exception err) { LOG.error("Error during responder execution", err); } } } else { LOG.info("Received task killed notification for task which is not currently being tracked: " + taskAttemptId); } }
LOG.info( "Attempting to send terminateRequest for fragment {} due to internal preemption invoked by {}", taskAttemptId.toString(), invokedByContainerEnd ? "containerEnd" : "taskEnd"); LlapNodeId nodeId = entityTracker.getNodeIdForTaskAttempt(taskAttemptId); constructQueryIdentifierProto( taskAttemptId.getTaskID().getVertexID().getDAGId().getId())) .setFragmentIdentifierString(taskAttemptId.toString()).build(); communicator.sendTerminateFragment(request, nodeId.getHostname(), nodeId.getPort(), new LlapProtocolClientProxy.ExecuteRequestCallback<TerminateFragmentResponseProto>() { LOG.info( "Not sending terminate request for fragment {} since it's node is not known. Already unregistered", taskAttemptId.toString());
String taskAttemptIdString = taskAttemptId.toString(); updateHeartbeatInfo(taskAttemptIdString);
String fragmentId = LlapTezUtils.stripAttemptPrefix(taskSpec.getTaskAttemptID().toString()); taskReporter = new LlapTaskReporter( completionListener,
/** * Submit the work for actual execution. */ public void submitWork(SubmitWorkRequestProto request, String llapHost, int llapPort) { // Register the pending events to be sent for this spec. VertexOrBinary vob = request.getWorkSpec(); assert vob.hasVertexBinary() != vob.hasVertex(); SignableVertexSpec vertex = null; try { vertex = vob.hasVertex() ? vob.getVertex() : SignableVertexSpec.parseFrom(vob.getVertexBinary()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } QueryIdentifierProto queryIdentifierProto = vertex.getQueryIdentifier(); TezTaskAttemptID attemptId = Converters.createTaskAttemptId(queryIdentifierProto, vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); final String fragmentId = attemptId.toString(); this.requestInfo = new RequestInfo(request, queryIdentifierProto, fragmentId, llapHost, llapPort); this.tezEvents = Lists.<TezEvent>newArrayList(); registerClient(); // Send out the actual SubmitWorkRequest final LlapTaskUmbilicalExternalClient client = this; communicator.start(); submitWork(); }
request.getFragmentNumber(), request.getAttemptNumber()).toString(); OutputStream socketStream = socket.getOutputStream(); LlapOutputSocketInitMessage.Builder builder =
Converters.createTaskAttemptId(vertex.getQueryIdentifier(), vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); if (LOG.isInfoEnabled()) { LOG.info("Queueing container for execution: fragemendId={}, {}",
.setIsGuaranteed(newState).setFragmentIdentifierString(attemptId.toString()) .setQueryIdentifier(constructQueryIdentifierProto( attemptId.getTaskID().getVertexID().getDAGId().getId())).build();
@Override public String toString() { return getID().toString(); }
@Override public String toString() { return "vertexName=" + vertexName + ", taskId=" + taskID + ", startTime=" + startTime + ", finishTime=" + finishTime + ", timeTaken=" + (finishTime - startTime) + ", status=" + state.name() + ", successfulAttemptID=" + (successfulAttemptID == null ? "null" : successfulAttemptID.toString()) + ", diagnostics=" + diagnostics + ", counters=" + ( tezCounters == null ? "null" : tezCounters.toString() .replaceAll("\\n", ", ").replaceAll("\\s+", " ")); }
@Override protected void handleTimeOut(TezTaskAttemptID attemptId) { eventHandler.handle(new TaskAttemptEventAttemptFailed(attemptId, TaskAttemptEventType.TA_TIMED_OUT, TaskFailureType.NON_FATAL, "AttemptID:" + attemptId.toString() + " Timed out after " + timeOut / 1000 + " secs", TaskAttemptTerminationCause.TASK_HEARTBEAT_ERROR)); } }
@Private public static void setHadoopCallerContext(HadoopShim hadoopShim, TezTaskAttemptID attemptID) { hadoopShim.setHadoopCallerContext("tez_ta:" + attemptID.toString()); }
public static Map<String, Object> convertDataEventDependecyInfoToATS(List<DataEventDependencyInfo> info) { ArrayList<Object> infoList = new ArrayList<Object>(); for (DataEventDependencyInfo event : info) { Map<String, Object> eventObj = new LinkedHashMap<String, Object>(); String id = ""; if (event.getTaskAttemptId() != null) { id = event.getTaskAttemptId().toString(); } eventObj.put(EntityTypes.TEZ_TASK_ATTEMPT_ID.name(), id); eventObj.put(ATSConstants.TIMESTAMP, event.getTimestamp()); infoList.add(eventObj); } Map<String,Object> object = new LinkedHashMap<String, Object>(); putInto(object, ATSConstants.LAST_DATA_EVENTS, infoList); return object; }
public static DataEventDependencyInfoProto toProto(DataEventDependencyInfo info) { DataEventDependencyInfoProto.Builder builder = DataEventDependencyInfoProto.newBuilder(); builder.setTimestamp(info.timestamp); if (info.taId != null) { builder.setTaskAttemptId(info.taId.toString()); } return builder.build(); }
public TaskAttemptStartedProto toProto() { TaskAttemptStartedProto.Builder builder = TaskAttemptStartedProto.newBuilder(); builder.setTaskAttemptId(taskAttemptId.toString()) .setStartTime(launchTime) .setContainerId(containerId.toString()) .setNodeId(nodeId.toString()); return builder.build(); }
public TaskFinishedProto toProto() { TaskFinishedProto.Builder builder = TaskFinishedProto.newBuilder(); builder.setTaskId(taskID.toString()) .setState(state.ordinal()) .setFinishTime(finishTime); if (diagnostics != null) { builder.setDiagnostics(diagnostics); } if (successfulAttemptID != null) { builder.setSuccessfulTaskAttemptId(successfulAttemptID.toString()); } return builder.build(); }
public static RecoveryProtos.EventMetaDataProto convertEventMetaDataToProto( EventMetaData eventMetaData) { RecoveryProtos.EventMetaDataProto.Builder builder = RecoveryProtos.EventMetaDataProto.newBuilder() .setProducerConsumerType(eventMetaData.getEventGenerator().ordinal()) .setEdgeVertexName(eventMetaData.getEdgeVertexName()) .setTaskVertexName(eventMetaData.getTaskVertexName()); if (eventMetaData.getTaskAttemptID() != null) { builder.setTaskAttemptId(eventMetaData.getTaskAttemptID().toString()); } return builder.build(); }
private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Events JSONArray events = new JSONArray(); JSONObject finishEvent = new JSONObject(); finishEvent.put(ATSConstants.TIMESTAMP, event.getFinishTime()); finishEvent.put(ATSConstants.EVENT_TYPE, HistoryEventType.TASK_FINISHED.name()); events.put(finishEvent); jsonObject.put(ATSConstants.EVENTS, events); JSONObject otherInfo = new JSONObject(); otherInfo.put(ATSConstants.START_TIME, event.getStartTime()); otherInfo.put(ATSConstants.FINISH_TIME, event.getFinishTime()); otherInfo.put(ATSConstants.TIME_TAKEN, (event.getFinishTime() - event.getStartTime())); otherInfo.put(ATSConstants.STATUS, event.getState().name()); otherInfo.put(ATSConstants.DIAGNOSTICS, event.getDiagnostics()); otherInfo.put(ATSConstants.COUNTERS, DAGUtils.convertCountersToJSON(event.getTezCounters())); if (event.getSuccessfulAttemptID() != null) { otherInfo.put(ATSConstants.SUCCESSFUL_ATTEMPT_ID, event.getSuccessfulAttemptID().toString()); } jsonObject.put(ATSConstants.OTHER_INFO, otherInfo); return jsonObject; }