public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public TaskSpec constructTaskSpec(DAG dag, String vertexName, int numSplits, ApplicationId appId, int index) { Vertex vertex = dag.getVertex(vertexName); ProcessorDescriptor processorDescriptor = vertex.getProcessorDescriptor(); List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs = vertex.getInputs(); List<RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor>> outputs = vertex.getOutputs(); Preconditions.checkState(inputs.size() == 1); Preconditions.checkState(outputs.size() == 1); List<InputSpec> inputSpecs = new ArrayList<>(); for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) { InputSpec inputSpec = new InputSpec(input.getName(), input.getIODescriptor(), 1); inputSpecs.add(inputSpec); } List<OutputSpec> outputSpecs = new ArrayList<>(); for (RootInputLeafOutput<OutputDescriptor, OutputCommitterDescriptor> output : outputs) { OutputSpec outputSpec = new OutputSpec(output.getName(), output.getIODescriptor(), 1); outputSpecs.add(outputSpec); } TezDAGID dagId = TezDAGID.getInstance(appId, 0); TezVertexID vertexId = TezVertexID.getInstance(dagId, 0); TezTaskID taskId = TezTaskID.getInstance(vertexId, index); TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); return new TaskSpec(taskAttemptId, dag.getName(), vertexName, numSplits, processorDescriptor, inputSpecs, outputSpecs, null); }
public String getName() { return dag.getName(); }
/** * Set the files etc that must be provided to the tasks of this DAG * @param localFiles * files that must be available locally for each task. These files * may be regular files, archives etc. as specified by the value * elements of the map. * @return {@link DAG} */ public synchronized DAG addTaskLocalFiles(Map<String, LocalResource> localFiles) { Preconditions.checkNotNull(localFiles); TezCommonUtils.addAdditionalLocalResources(localFiles, commonTaskLocalFiles, "DAG " + getName()); return this; }
tezClient = TezSessionManager.getClient(conf, requestAMResources, dag.getCredentials(), tezJobConf); log.info("Submitting DAG " + dag.getName()); dagClient = tezClient.submitDAG(dag); appId = tezClient.getAppMasterApplicationId(); log.info("Submitted DAG " + dag.getName() + ". Application id: " + appId); } catch (Exception e) { if (tezClient != null) {
public boolean notifyFinishedOrFailed() { DAGStatus dagStatus = runningJob.getDAGStatus(); if (dagStatus == null) { return false; } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { Map<Enum, Long> warningAggMap = new HashMap<Enum, Long>(); DAG dag = runningJob.getDAG(); for (Vertex v : dag.getVertices()) { TezVertexStats tts = tezStats.getVertexStats(dag.getName(), v.getName()); if (tts == null) { continue; //vertex groups } Map<String, Map<String, Long>> counterGroups = tts.getCounters(); if (counterGroups == null) { log.warn("Counters are not available for vertex " + v.getName() + ". Not computing warning aggregates."); } else { computeWarningAggregate(counterGroups, warningAggMap); } } if (aggregateWarning) { CompilationMessageCollector.logAggregate(warningAggMap, MessageType.Warning, log); } return true; } return false; } }
"Tez Example DAG: " + dag.getName()); ApplicationId appId = tezClientInternal.getAppMasterApplicationId(); if (hadoopShim == null) {
ApplicationSubmissionContext appContext = TezClientUtils .createApplicationSubmissionContext( appId, dag, dag.getName(), amConfig, tezJarResources, credentials, usingTezArchiveDeploy, apiVersionInfo, servicePluginsDescriptor, javaOptsChecker); String callerContextStr = ""; + ", dagName=" + dag.getName() + callerContextStr);
hdfsBytesWritten = fsGrp.findCounter(PigStatsUtil.HDFS_BYTES_WRITTEN).getValue(); } else { LOG.warn("Failed to get counters for DAG: " + dag.getName());