private String constructLlapLogUrl(final TezTaskAttemptID attemptID, final String containerIdString, final boolean isDone, final String nmAddress) { String dagId = attemptID.getTaskID().getVertexID().getDAGId().toString(); String filename = JOINER.join(currentHiveQueryId, "-", dagId, ".log", (isDone ? ".done" : ""), "?nm.id=", nmAddress); String url = PATH_JOINER.join(timelineServerUri, "ws", "v1", "applicationhistory", "containers", containerIdString, "logs", filename); return url; }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }
@Override public void toSummaryProtoStream(OutputStream outputStream) throws IOException { SummaryEventProto.Builder builder = RecoveryProtos.SummaryEventProto.newBuilder() .setDagId(vertexID.getDAGId().toString()) .setTimestamp(commitStartTime) .setEventType(getEventType().ordinal()) .setEventPayload( ByteString.copyFrom(vertexID.toString().getBytes(charSet))); builder.build().writeDelimitedTo(outputStream); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static org.apache.hadoop.mapred.TaskID createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getDAGId().getApplicationId(); return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp()) + String.valueOf(vId.getId()), appId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId()); }
Map<Vertex, Edge> out = v.getOutputVertices(); if (out == null) { result.put(v.getVertexId().getId(), Sets.newHashSet()); } else { Set<Integer> set = Sets.newHashSetWithExpectedSize(vc); for (Vertex outV : out.keySet()) { set.add(outV.getVertexId().getId()); result.put(v.getVertexId().getId(), set); while (!queue.isEmpty()) { TezVertexID id = queue.poll(); if (processed.contains(id.getId())) continue; // Already processed. See backtracking. Vertex v = dag.getVertex(id); Map<Vertex, Edge> out = v.getOutputVertices(); for (Vertex outV : out.keySet()) { TezVertexID outId = outV.getVertexId(); int outNum = outId.getId(); if (!processed.contains(outNum)) { if (!doBacktrack) { int num = id.getId(); processed.add(num); Set<Integer> deps = result.get(num); result.get(inV.getVertexId().getId()).addAll(deps);
@Override public String apply(TezVertexID vertexId) { return vertexId.toString(); } });
final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID); Vertex vertex = currentDAG.getVertex(tezVertexID); if (vertex == null) { result.put(VERTEX_PROGRESS, new ProgressInfo(tezVertexID.toString(), vertex.getCompletedTaskProgress())); renderJSON(result);
/** * Cleans entries from the object registry, and resets the startedInputsMap if required * * @param containerTask * the new task specification. Must be a valid task */ private void cleanupOnTaskChanged(ContainerTask containerTask) { Preconditions.checkState(!containerTask.shouldDie()); Preconditions.checkState(containerTask.getTaskSpec() != null); TezVertexID newVertexID = containerTask.getTaskSpec().getTaskAttemptID().getTaskID() .getVertexID(); if (lastVertexID != null) { if (!lastVertexID.equals(newVertexID)) { objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX); } if (!lastVertexID.getDAGId().equals(newVertexID.getDAGId())) { objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG); startedInputsMap = HashMultimap.create(); } } lastVertexID = newVertexID; }
case Constants.TEZ_VERTEX_ID: String vertexName = entity; TezVertexID tezVertexID = TezVertexID.fromString(vertexName); if (!tezDAGID.equals(tezVertexID.getDAGId())) { LOG.warn(vertexName + " does not belong to " + tezDAGID); continue; String taskName = entity; TezTaskID tezTaskID = TezTaskID.fromString(taskName); if (!tezDAGID.equals(tezTaskID.getVertexID().getDAGId())) { LOG.warn(taskName + " does not belong to " + tezDAGID); continue; String taskAttemptName = entity; TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName); if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) { LOG.warn(taskAttemptName + " does not belong to " + tezDAGID); continue;
@Override public TezVertexID apply(String input) { return TezVertexID.fromString(input); } });
@Override public boolean equals(Object o) { if (this == o) { return true; } if(o == null) { return false; } if (o.getClass() == this.getClass()) { VertexIdentifierImpl other = (VertexIdentifierImpl) o; return this.vertexId.equals(other.vertexId); } else { return false; } }
/**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/ @Override public int compareTo(TezID o) { TezTaskID that = (TezTaskID)o; int vertexComp = this.vertexId.compareTo(that.vertexId); if(vertexComp == 0) { return this.id - that.id; } else return vertexComp; } @Override
private static TimelineEntity convertVertexReconfigureDoneEvent( VertexConfigurationDoneEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getVertexID().toString()); atsEntity.setEntityType(EntityTypes.TEZ_VERTEX_ID.name()); atsEntity.addPrimaryFilter(ATSConstants.APPLICATION_ID, event.getVertexID().getDAGId().getApplicationId().toString()); atsEntity.addPrimaryFilter(EntityTypes.TEZ_DAG_ID.name(), event.getVertexID().getDAGId().toString()); TimelineEvent updateEvt = new TimelineEvent(); updateEvt.setEventType(HistoryEventType.VERTEX_CONFIGURE_DONE.name()); updateEvt.setTimestamp(event.getReconfigureDoneTime()); Map<String,Object> eventInfo = new HashMap<String, Object>(); if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) { Map<String, Object> updatedEdgeManagers = new HashMap<String, Object>(); for (Entry<String, EdgeProperty> entry : event.getSourceEdgeProperties().entrySet()) { updatedEdgeManagers.put(entry.getKey(), DAGUtils.convertEdgeProperty(entry.getValue())); } eventInfo.put(ATSConstants.UPDATED_EDGE_MANAGERS, updatedEdgeManagers); } eventInfo.put(ATSConstants.NUM_TASKS, event.getNumTasks()); updateEvt.setEventInfo(eventInfo); atsEntity.addEvent(updateEvt); atsEntity.addOtherInfo(ATSConstants.NUM_TASKS, event.getNumTasks()); return atsEntity; }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
private static int vertexNum(TaskInfo taskInfo) { return taskInfo.getAttemptId().getTaskID().getVertexID().getId(); // Sigh... }
@Override public String apply(TezVertexID vertexId) { return vertexId.toString(); } });
TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, getContext().getTaskVertexIndex()); TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber()); jobConf.set(MRInput.TEZ_MAPREDUCE_DAG_ID, tezDAGID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_VERTEX_ID, tezVertexID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ID, tezTaskID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_TASK_ATTEMPT_ID, tezTaskAttemptID.toString());
@Override public TezVertexID apply(String input) { return TezVertexID.fromString(input); } });