private void updateMapRedStatsJson(MapRedStats stats, RunningJob rj) throws IOException, JSONException { if (statsJSON == null) { statsJSON = new JSONObject(); } if (stats != null) { if (stats.getNumMap() >= 0) { statsJSON.put(NUMBER_OF_MAPPERS, stats.getNumMap()); } if (stats.getNumReduce() >= 0) { statsJSON.put(NUMBER_OF_REDUCERS, stats.getNumReduce()); } if (stats.getCounters() != null) { statsJSON.put(COUNTERS, getCountersJson(stats.getCounters())); } } if (rj != null) { statsJSON.put(JOB_ID, rj.getID().toString()); statsJSON.put(JOB_FILE, rj.getJobFile()); statsJSON.put(TRACKING_URL, rj.getTrackingURL()); statsJSON.put(MAP_PROGRESS, Math.round(rj.mapProgress() * 100)); statsJSON.put(REDUCE_PROGRESS, Math.round(rj.reduceProgress() * 100)); statsJSON.put(CLEANUP_PROGRESS, Math.round(rj.cleanupProgress() * 100)); statsJSON.put(SETUP_PROGRESS, Math.round(rj.setupProgress() * 100)); statsJSON.put(COMPLETE, rj.isComplete()); statsJSON.put(SUCCESSFUL, rj.isSuccessful()); } }
jobConf.set("mapred.tip.id", taId.getTaskID().toString()); jobConf.set("mapred.task.id", taId.toString()); jobConf.set("mapred.job.id", jobID.toString()); amClient.reportMapedTaskID(containerId, taId.toString()); RecordWriter writer = outputFormat.getRecordWriter(dfs, jobConf, "part-r", Reporter.NULL);
public List<JobItemBean> listJobs(String user, boolean showall, String jobId, int numRecords, boolean showDetails) throws NotAuthorizedException, BadParam, IOException, InterruptedException { UserGroupInformation ugi = null; WebHCatJTShim tracker = null; ArrayList<String> ids = new ArrayList<String>(); try { ugi = UgiFactory.getUgi(user); tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); JobStatus[] jobs = tracker.getAllJobs(); if (jobs != null) { for (JobStatus job : jobs) { String id = job.getJobID().toString(); if (showall || user.equals(job.getUsername())) ids.add(id); } } } catch (IllegalStateException e) { throw new BadParam(e.getMessage()); } finally { if (tracker != null) tracker.close(); if (ugi != null) FileSystem.closeAllForUGI(ugi); } return getJobStatus(ids, user, showall, jobId, numRecords, showDetails); }
String jobId = t.getTaskAttemptId().getJobID().toString(); if (firstError) { console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId);
String jobId = t.getTaskAttemptId().getJobID().toString(); if (firstError) { console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId);
RunningJob rj = new JobClient(job).submitJob(job); LOG.info("Submitted compaction job '" + job.getJobName() + "' with jobID=" + rj.getID() + " compaction ID=" + id); txnHandler.setHadoopJobId(rj.getID().toString(), id); rj.waitForCompletion(); if (!rj.isSuccessful()) {
/** * Create a new QueueStatusBean * * @param state store job state * @param status job status * @param profile job profile */ public QueueStatusBean(JobState state, JobStatus status, JobProfile profile) throws IOException { this.status = status; this.profile = profile; id = profile.getJobID().toString(); parentId = state.getParent(); percentComplete = state.getPercentComplete(); exitValue = state.getExitValue(); user = profile.getUser(); callback = state.getCallback(); completed = state.getCompleteStatus(); userargs = state.getUserArgs(); } }
TaskReport[] reduceTaskReport ) throws IOException { jobId = runningJob.getID().toString(); jobName = runningJob.getJobName(); trackingURL = runningJob.getTrackingURL();
for (ClientStatsPublisher clientStatPublisher : clientStatPublishers) { try { clientStatPublisher.run(exctractedCounters, rj.getID().toString()); } catch (RuntimeException runtimeException) { LOG.error("Exception " + runtimeException.getClass().getCanonicalName() MapRedStats mapRedStats = new MapRedStats(numMap, numReduce, cpuMsec, success, rj.getID().toString()); mapRedStats.setCounters(ctrs);
job, numMap, numReduce, cpuMsec, false, rj.getID().toString()); updateMapRedTaskWebUIStatistics(mapRedStats, rj); for (ClientStatsPublisher clientStatPublisher : clientStatPublishers) { try { clientStatPublisher.run(exctractedCounters, rj.getID().toString()); } catch (RuntimeException runtimeException) { LOG.error("Exception " + runtimeException.getClass().getCanonicalName()
killJob(); jobID = rj.getID().toString();
/** * from StreamJob.java. */ public void jobInfo(RunningJob rj) { if (ShimLoader.getHadoopShims().isLocalMode(job)) { console.printInfo("Job running in-process (local Hadoop)"); } else { if (SessionState.get() != null) { SessionState.get().getHiveHistory().setTaskProperty(queryId, getId(), Keys.TASK_HADOOP_ID, rj.getID().toString()); } console.printInfo(getJobStartMsg(rj.getID()) + ", Tracking URL = " + rj.getTrackingURL()); console.printInfo("Kill Command = " + HiveConf.getVar(job, ConfVars.MAPREDBIN) + " job -kill " + rj.getID()); } }
rj.killJob(); jobID = rj.getID().toString();
/** * from StreamJob.java. */ public void jobInfo(RunningJob rj) { if (ShimLoader.getHadoopShims().isLocalMode(job)) { console.printInfo("Job running in-process (local Hadoop)"); } else { if (SessionState.get() != null) { SessionState.get().getHiveHistory().setTaskProperty(queryId, getId(), Keys.TASK_HADOOP_ID, rj.getID().toString()); } console.printInfo(getJobStartMsg(rj.getID()) + ", Tracking URL = " + rj.getTrackingURL()); console.printInfo("Kill Command = " + HiveConf.getVar(job, HiveConf.ConfVars.HADOOPBIN) + " job -kill " + rj.getID()); } }
when(jobID.toString()).thenReturn(TEST_JOB_ID);
/** * @deprecated use getJobID() instead */ @Deprecated public String getJobId() { return jobid.toString(); }
/** * Get the done summary file name for a job. * @param jobId the jobId. * @return the conf file name. */ public static String getIntermediateSummaryFileName(JobId jobId) { return TypeConverter.fromYarn(jobId).toString() + SUMMARY_FILE_NAME_SUFFIX; }
/** * Get the job history file path for non Done history files. */ public static Path getStagingJobHistoryFile(Path dir, JobId jobId, int attempt) { return getStagingJobHistoryFile(dir, TypeConverter.fromYarn(jobId).toString(), attempt); }
/** * Localize the given JobConf to be specific for this task. */ public void localizeConfiguration(JobConf conf) throws IOException { conf.set(JobContext.TASK_ID, taskId.getTaskID().toString()); conf.set(JobContext.TASK_ATTEMPT_ID, taskId.toString()); conf.setBoolean(JobContext.TASK_ISMAP, isMapTask()); conf.setInt(JobContext.TASK_PARTITION, partition); conf.set(JobContext.ID, taskId.getJobID().toString()); }