public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) { // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside. return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ctx.getApplicationId(), ctx.getDagIdentifier()), ctx.getTaskVertexIndex()), ctx.getTaskIndex()), ctx.getTaskAttemptNumber()); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) { return TezTaskAttemptID.getInstance(taskId, id); }
@Override public long getFirstAttemptStartTime() { readLock.lock(); try { // The first attempt will always have an index of 0. return getAttempt(TezTaskAttemptID.getInstance(getTaskId(), 0)).getScheduleTime(); } finally { readLock.unlock(); } }
public static TezTaskAttemptID fromString(String taIdStr) { try { int pos1 = taIdStr.indexOf(SEPARATOR); int pos2 = taIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taIdStr.indexOf(SEPARATOR, pos4 + 1); int pos6 = taIdStr.indexOf(SEPARATOR, pos5 + 1); String rmId = taIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taIdStr.substring(pos4 + 1, pos5)); int taskId = Integer.parseInt(taIdStr.substring(pos5 + 1, pos6)); int id = Integer.parseInt(taIdStr.substring(pos6 + 1)); return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), taskId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezTaskAttemptID readTezTaskAttemptID(DataInput in) throws IOException { TezTaskID taskID = TezTaskID.readTezTaskID(in); int attemptIdInt = TezID.readID(in); return getInstance(taskID, attemptIdInt); }
public static TezTaskAttemptID fromMRTaskAttemptId( org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptId) { return TezTaskAttemptID.getInstance( fromMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); }
getAttempt(TezTaskAttemptID.getInstance(task.getTaskId(), indexes.get(2))); if(attempt == null) { continue;
@Override public void sendEventToProcessor(Collection<CustomProcessorEvent> events, int taskId) { checkAndThrowIfDone(); Preconditions.checkArgument(taskId >= 0 && taskId < managedVertex.getTotalTasks(), "Invalid taskId " + taskId + "; " + "There are " + managedVertex.getTotalTasks() + " tasks in total."); if (events != null && events.size() > 0) { List<TezEvent> tezEvents = new ArrayList<>(); for (CustomProcessorEvent event : events) { TezEvent tezEvent = new TezEvent(event, null); // use dummy task attempt id since this is not an task attempt specific event and task // attempt id won't be used anyway EventMetaData destinationMeta = new EventMetaData(EventProducerConsumerType.PROCESSOR, managedVertex.getName(), managedVertex.getName(), TezTaskAttemptID.getInstance(managedVertex.getTask(taskId).getTaskId(), -1)); tezEvent.setDestinationInfo(destinationMeta); tezEvents.add(tezEvent); } appContext.getEventHandler().handle( new VertexEventRouteEvent(managedVertex.getVertexId(), tezEvents)); } }
TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, taskAttemptIndex); sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId,
TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, getContext().getTaskVertexIndex()); TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber()); jobConf.set(MRInput.TEZ_MAPREDUCE_DAG_ID, tezDAGID.toString()); jobConf.set(MRInput.TEZ_MAPREDUCE_VERTEX_ID, tezVertexID.toString());