public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public static TezTaskAttemptID createTaskAttemptId( QueryIdentifierProto queryIdProto, int vertexIndex, int fragmentNum, int attemptNum) { // Come ride the API roller-coaster! return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ConverterUtils.toApplicationId( queryIdProto.getApplicationIdString()), queryIdProto.getDagIndex()), vertexIndex), fragmentNum), attemptNum); }
public static TezTaskAttemptID createTaskAttemptId(TaskContext ctx) { // Come ride the API roller-coaster #2! The best part is that ctx has TezTaskAttemptID inside. return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance( ctx.getApplicationId(), ctx.getDagIdentifier()), ctx.getTaskVertexIndex()), ctx.getTaskIndex()), ctx.getTaskAttemptNumber()); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
TezDAGID tezDagId = TezDAGID.getInstance(appId, 300); TezVertexID tezVertexId = TezVertexID.getInstance(tezDagId, 400); TezTaskID tezTaskId = TezTaskID.getInstance(tezVertexId, 500);
boolean isGuaranteed) { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return SubmitWorkRequestProto
public static TezDAGID fromString(String dagId) { int id = 0; int appId = 0; String[] split = dagId.split("_"); if (split.length != 4 || !dagId.startsWith(DAG + "_")) { throw new IllegalArgumentException("Invalid DAG Id format : " + dagId); } String rmId = split[1]; try { appId = Integer.parseInt(split[2]); } catch (NumberFormatException e) { throw new IllegalArgumentException("Error while parsing App Id '" + split[2] + "' of DAG Id : " + dagId); } try { id = Integer.parseInt(split[3]); } catch (NumberFormatException e) { throw new IllegalArgumentException("Error while parsing Id '" + split[3] + "' of DAG Id : " + dagId); } return TezDAGID.getInstance(rmId, appId, id); }
public static TezDAGID readTezDAGID(DataInput in) throws IOException { long clusterId = in.readLong(); int appId = in.readInt(); int dagIdInt = TezID.readID(in); TezDAGID dagID = getInstance(ApplicationId.newInstance(clusterId, appId), dagIdInt); return dagID; }
public static TezVertexID fromString(String vertexIdStr) { try { int pos1 = vertexIdStr.indexOf(SEPARATOR); int pos2 = vertexIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = vertexIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = vertexIdStr.indexOf(SEPARATOR, pos3 + 1); String rmId = vertexIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(vertexIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(vertexIdStr.substring(pos3 + 1, pos4)); int id = Integer.parseInt(vertexIdStr.substring(pos4 + 1)); return TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezTaskID fromString(String taskIdStr) { try { int pos1 = taskIdStr.indexOf(SEPARATOR); int pos2 = taskIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taskIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taskIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taskIdStr.indexOf(SEPARATOR, pos4 + 1); String rmId = taskIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taskIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taskIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taskIdStr.substring(pos4 + 1, pos5)); int id = Integer.parseInt(taskIdStr.substring(pos5 + 1)); return TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezTaskAttemptID fromString(String taIdStr) { try { int pos1 = taIdStr.indexOf(SEPARATOR); int pos2 = taIdStr.indexOf(SEPARATOR, pos1 + 1); int pos3 = taIdStr.indexOf(SEPARATOR, pos2 + 1); int pos4 = taIdStr.indexOf(SEPARATOR, pos3 + 1); int pos5 = taIdStr.indexOf(SEPARATOR, pos4 + 1); int pos6 = taIdStr.indexOf(SEPARATOR, pos5 + 1); String rmId = taIdStr.substring(pos1 + 1, pos2); int appId = Integer.parseInt(taIdStr.substring(pos2 + 1, pos3)); int dagId = Integer.parseInt(taIdStr.substring(pos3 + 1, pos4)); int vId = Integer.parseInt(taIdStr.substring(pos4 + 1, pos5)); int taskId = Integer.parseInt(taIdStr.substring(pos5 + 1, pos6)); int id = Integer.parseInt(taIdStr.substring(pos6 + 1)); return TezTaskAttemptID.getInstance( TezTaskID.getInstance( TezVertexID.getInstance( TezDAGID.getInstance(rmId, appId, dagId), vId), taskId), id); } catch (Exception e) { e.printStackTrace(); } return null; }
public static TezDAGID fromMRJobId( org.apache.hadoop.mapreduce.JobID jobId) { return TezDAGID.getInstance(ApplicationId.newInstance( Long.parseLong(jobId.getJtIdentifier()), jobId.getId()), 1); }
dagId = TezDAGID.getInstance(appAttemptID.getApplicationId(), dagCounter.incrementAndGet());
jobConf.setInt(MRInput.TEZ_MAPREDUCE_DAG_ATTEMPT_NUMBER, getContext().getDAGAttemptNumber()); TezDAGID tezDAGID = TezDAGID.getInstance(getContext().getApplicationId(), getContext().getDagIdentifier()); TezVertexID tezVertexID = TezVertexID.getInstance(tezDAGID, getContext().getTaskVertexIndex()); TezTaskID tezTaskID = TezTaskID.getInstance(tezVertexID, getContext().getTaskIndex());