public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) { // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside. return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ctx.getApplicationId(), ctx.getDagIdentifier()), ctx.getTaskVertexIndex()), ctx.getTaskIndex()), ctx.getTaskAttemptNumber()); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }
TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500); TezTaskAttemptID tezTaskAttemptId = TezTaskAttemptID.getInstance(tezTaskId, 600);
public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) { return TezVertexID.getInstance(dagId, vertexId); }
ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return SubmitWorkRequestProto .newBuilder()
public static TezVertexID fromString(String vertexIdStr) { try { int pos1 = vertexIdStr.indexOf(SEPARATOR); int pos2 = vertexIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = vertexIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = vertexIdStr.indexOf(SEPARATOR, pos3 + 1); String rmId = vertexIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(vertexIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(vertexIdStr.substring(pos3 + 1, pos4)); int id = Integer.parseInt(vertexIdStr.substring(pos4 + 1)); return TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
Collection<Vertex> getVerticesByIdx(DAG dag, Collection<Integer> indexes) { Collection<Vertex> vertices = new ArrayList<Vertex>(indexes.size()); final TezDAGID tezDAGID = dag.getID(); for (Integer idx : indexes) { final TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, idx); if (tezVertexID == null) { continue; } final Vertex vertex = dag.getVertex(tezVertexID); if (vertex != null) { vertices.add(vertex); } } return vertices; }
public static TezTaskID fromString(String taskIdStr) { try { int pos1 = taskIdStr.indexOf(SEPARATOR); int pos2 = taskIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taskIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taskIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taskIdStr.indexOf(SEPARATOR, pos4 + 1); String rmId = taskIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taskIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taskIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taskIdStr.substring(pos4 + 1, pos5)); int id = Integer.parseInt(taskIdStr.substring(pos5 + 1)); return TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
Vertex getVertexFromIndex(DAG dag, Integer vertexIndex) { final TezVertexID tezVertexID = TezVertexID.getInstance(dag.getID(), vertexIndex); Vertex vertex = dag.getVertex(tezVertexID); return vertex; }
public static TezTaskAttemptID fromString(String taIdStr) { try { int pos1 = taIdStr.indexOf(SEPARATOR); int pos2 = taIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taIdStr.indexOf(SEPARATOR, pos4 + 1); int pos6 = taIdStr.indexOf(SEPARATOR, pos5 + 1); String rmId = taIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taIdStr.substring(pos4 + 1, pos5)); int taskId = Integer.parseInt(taIdStr.substring(pos5 + 1, pos6)); int id = Integer.parseInt(taIdStr.substring(pos6 + 1)); return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), taskId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezVertexID readTezVertexID(DataInput in) throws IOException { TezDAGID dagID = TezDAGID.readTezDAGID(in); int vertexIdInt = TezID.readID(in); return getInstance(dagID, vertexIdInt); }
public static TezTaskID fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) { return TezTaskID.getInstance( TezVertexID.getInstance(fromMRJobId(taskid.getJobID()), (taskid.getTaskType() == TaskType.MAP ? 0 : 1) ), taskid.getId()); }
final TezVertexID tezVertexID = TezVertexID.getInstance(currentDAG.getID(), vertexID); Vertex vertex = currentDAG.getVertex(tezVertexID); if (vertex == null) {
TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, getContext().getTaskVertexIndex()); TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex()); TezTaskAttemptID tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, getContext().getTaskAttemptNumber());