synchronized void addToCache(JobInProgress job) { RetireJobInfo info = new RetireJobInfo(job.getStatus(), job.getProfile(), job.getFinishTime(), job.getHistoryFile()); jobRetireInfoQ.add(info); jobIDStatusMap.put(info.status.getJobID(), info); if (jobRetireInfoQ.size() > retiredJobsCacheSize) { RetireJobInfo removed = jobRetireInfoQ.remove(); jobIDStatusMap.remove(removed.status.getJobID()); LOG.info("Retired job removed from cache " + removed.status.getJobID()); } }
/** * Adds a job to the jobtracker. Make sure that the checks are inplace before * adding a job. This is the core job submission logic * @param jobId The id for the job submitted which needs to be added */ synchronized JobStatus addJob(JobID jobId, JobInProgress job) { totalSubmissions++; synchronized (jobs) { synchronized (taskScheduler) { jobs.put(job.getProfile().getJobID(), job); for (JobInProgressListener listener : jobInProgressListeners) { try { listener.jobAdded(job); } catch (IOException ioe) { LOG.warn("Failed to add and so skipping the job : " + job.getJobID() + ". Exception : " + ioe); } } } } myInstrumentation.submitJob(job.getJobConf(), jobId); LOG.info("Job " + jobId + " added successfully for user '" + job.getJobConf().getUser() + "' to queue '" + job.getJobConf().getQueueName() + "'"); return job.getStatus(); }
public int compare(JobInProgress o1, JobInProgress o2) { int res = o1.getPriority().compareTo(o2.getPriority()); if(res == 0) { if(o1.getStartTime() < o2.getStartTime()) res = -1; else res = (o1.getStartTime()==o2.getStartTime() ? 0 : 1); } return res; } };
/** * Have all the tasks of type specified by isMap finished in this job? * * @param jip * @param isMap * @return true if finished, false otherwise */ static boolean haveAllTasksFinished(JobInProgress jip, boolean isMap) { return ((isMap ? jip.runningMaps() : jip.runningReduces()) == 0); }
private boolean shouldRunOnTaskTracker(String taskTracker) { // // Check if too many tasks of this job have failed on this // tasktracker prior to assigning it a new one. // int taskTrackerFailedTasks = getTrackerTaskFailures(taskTracker); if ((flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) && taskTrackerFailedTasks >= maxTaskFailuresPerTracker) { if (LOG.isDebugEnabled()) { String flakyTracker = convertTrackerNameToHostName(taskTracker); LOG.debug("Ignoring the black-listed tasktracker: '" + flakyTracker + "' for assigning a new task"); } return false; } return true; }
for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext(); ) { JobInProgress job = it.next(); totalMaps += job.desiredMaps(); totalReduces += job.desiredReduces(); comMaps += job.finishedMaps(); comReduces += job.finishedReduces(); if (isRunning) { totalRunningMaps += job.runningMaps(); totalRunningReduces += job.runningReduces(); for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext(); ++rowId) { JobInProgress job = it.next(); JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String runningMapTableData = (isRunning) ? job.runningMaps() + "</td><td>" : ""; String runningReduceTableData = (isRunning) ? job.runningReduces() + "</td><td>" : ""; String jobpri = job.getPriority().toString(); String schedulingInfo = job.getStatus().getSchedulingInfo();
JobStatus status = job.getStatus(); JobProfile profile = job.getProfile(); String user = StringUtils.escapeString(profile.getUser(), StringUtils.ESCAPE_CHAR, StringUtils.ESCAPE_CHAR, charsToEscape); Counters jobCounters = job.getJobCounters(); long mapSlotSeconds = (jobCounters.getCounter(Counter.SLOTS_MILLIS_MAPS) + jobCounters.getCounter(Counter.FALLOW_SLOTS_MILLIS_REDUCES)) / 1000; LOG.info("jobId=" + job.getJobID() + StringUtils.COMMA + "submitTime" + EQUALS + job.getStartTime() + StringUtils.COMMA + "launchTime" + EQUALS + job.getLaunchTime() + StringUtils.COMMA + "finishTime" + EQUALS + job.getFinishTime() + StringUtils.COMMA + "numMaps" + EQUALS + job.getTasks(TaskType.MAP).length + StringUtils.COMMA + "numSlotsPerMap" + EQUALS + job.getNumSlotsPerMap() + StringUtils.COMMA + "numReduces" + EQUALS + job.getTasks(TaskType.REDUCE).length + StringUtils.COMMA + "numSlotsPerReduce" + EQUALS + job.getNumSlotsPerReduce() + StringUtils.COMMA + "user" + EQUALS + user + StringUtils.COMMA +
for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext();) { JobInProgress job = it.next(); JobProfile profile = job.getProfile(); String user = profile.getUser(); String name = profile.getJobName(). replace(' ', '_').replace('\t', '_').replace('\n', '_'); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int runningMaps = 0; int failedMaps = 0; int killedMaps = 0; for (TaskInProgress tip: job.getTasks(TaskType.MAP)) { if (tip.isRunning()) { runningMaps += tip.getActiveTasks().size(); int failedReduces = 0; int killedReduces = 0; for (TaskInProgress tip: job.getTasks(TaskType.REDUCE)) { if (tip.isRunning()) { runningReduces += tip.getActiveTasks().size(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); int nonLocalRunningMaps = job.getNonLocalRunningMaps().size(); long submitTime = job.getStartTime(); long finishTime = job.getFinishTime(); String jobpri = job.getPriority().toString(); JobID jobId = job.getJobID(); double mem = 0, memMax = 0, memMaxPeak = 0, memCost = 0;
for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext(); ++rowId) { JobInProgress job = it.next(); JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String name = profile.getJobName(); String jobpri = job.getPriority().toString(); String schedulingInfo = job.getStatus().getSchedulingInfo(); if(schedulingInfo.trim().equals("")){ schedulingInfo = " ";
JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); int runState = status.getRunState(); int flakyTaskTrackers = job.getNoOfBlackListedTrackers(); out.print("<b>User:</b> " + HtmlQuoting.quoteHtmlChars(profile.getUser()) + "<br>\n"); profile.getJobFile() + "</a><br>\n"); out.print("<b>Submit Host:</b> " + HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostName()) + "<br>\n"); out.print("<b>Submit Host Address:</b> " + HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostAddress()) + "<br>\n"); job.getTasks(TaskType.JOB_SETUP)); out.print("<br>\n"); if (runState == JobStatus.RUNNING) { out.print("<b>Status:</b> Running<br>\n"); out.print("<b>Started at:</b> " + new Date(job.getStartTime()) + "<br>\n"); out.print("<b>Running for:</b> " + StringUtils.formatTimeDiff( System.currentTimeMillis(), job.getStartTime()) + "<br>\n"); } else { if (runState == JobStatus.SUCCEEDED) { out.print("<b>Status:</b> Succeeded<br>\n"); out.print("<b>Started at:</b> " + new Date(job.getStartTime()) + "<br>\n"); out.print("<b>Finished at:</b> " + new Date(job.getFinishTime()) + "<br>\n"); out.print("<b>Finished in:</b> " + StringUtils.formatTimeDiff( job.getFinishTime(), job.getStartTime()) + "<br>\n"); } else if (runState == JobStatus.FAILED) {
"match the expected value", jip.getStartTime() == jobInfo.getSubmitTime()); "match the expected value", jip.getLaunchTime() == jobInfo.getLaunchTime()); "match the expected value", jip.getFinishTime() == jobInfo.getFinishTime()); "match the expected value", jip.getPriority().toString().equals( jobInfo.getPriority())); Counters c = new Counters(jip.getCounters()); Counters jiCounters = jobInfo.getTotalCounters(); assertTrue("Counters of job obtained from history file did not " + jobInfo.getTotalMaps() == jip.desiredMaps()); jobInfo.getTotalReduces() == jip.desiredReduces()); jobInfo.getFinishedMaps() == jip.finishedMaps()); jobInfo.getFinishedReduces() == jip.finishedReduces());
for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() != JobStatus.RUNNING) { continue; && (job.runningMaps() + job.runningReduces() >= maxTasksPerJob)) { continue; task = job.obtainNewMapTask(taskTracker, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts()); task = job.obtainNewReduceTask(taskTracker, numTaskTrackers, taskTrackerManager.getNumberOfUniqueHosts());
if (tasksInited.get() || isComplete()) { return; logSubmissionToJobHistory(); setPriority(this.priority); generateAndStoreTokens(); TaskSplitMetaInfo[] taskSplitMetaInfo = createSplits(jobId); numMapTasks = taskSplitMetaInfo.length; checkTaskLimits(); jobtracker.getInstrumentation().addWaitingMaps(getJobID(), numMapTasks); jobtracker.getInstrumentation().addWaitingReduces(getJobID(), numReduceTasks); createMapTasks(jobFile.toString(), taskSplitMetaInfo); nonRunningMapCache = createCache(taskSplitMetaInfo, maxLevel); createReduceTasks(jobFile.toString()); numMapTasks)); initSetupCleanupTasks(jobFile.toString());
static void checkTaskCounts(JobInProgress jip, int runningMaps, int pendingMaps, int runningReduces, int pendingReduces) { Counters counter = jip.getJobCounters(); long totalTaskCount = counter.getCounter(JobCounter.TOTAL_LAUNCHED_MAPS) + counter.getCounter(JobCounter.TOTAL_LAUNCHED_REDUCES); LOG.info("totalTaskCount is " + totalTaskCount); LOG.info(" Running Maps:" + jip.runningMaps() + " Pending Maps:" + jip.pendingMaps() + " Running Reds:" + jip.runningReduces() + " Pending Reds:" + jip.pendingReduces()); assertEquals(jip.getNumTaskCompletionEvents(),totalTaskCount); assertEquals(runningMaps, jip.runningMaps()); assertEquals(pendingMaps, jip.pendingMaps()); assertEquals(runningReduces, jip.runningReduces()); assertEquals(pendingReduces, jip.pendingReduces()); }
public void initJob(JobInProgress job) { try { JobStatus prevStatus = (JobStatus)job.getStatus().clone(); job.initTasks(); if (job.isJobEmpty()) { completeEmptyJob(job); } else if (!job.isSetupCleanupRequired()) { job.completeSetup(); } JobStatus newStatus = (JobStatus)job.getStatus().clone(); if (prevStatus.getRunState() != newStatus.getRunState()) { JobStatusChangeEvent event = new JobStatusChangeEvent(job, EventType.RUN_STATE_CHANGED, prevStatus, newStatus); for (JobInProgressListener listener : listeners) { listener.jobUpdated(event); } } } catch (KillInterruptedException kie) { killJob(job.getJobID()); } catch (IOException ioe) { failJob(job); } }
for (JobID id : jobsToRecover) { JobInProgress job = new JobInProgress(id, JobTracker.this, conf); JobHistory.JobInfo.getJobHistoryFileName(job.getJobConf(), id); Path jobHistoryFilePath = JobHistory.JobInfo.getJobHistoryLogLocation(logFileName); JobHistory.JobInfo.recoverJobHistoryFile(job.getJobConf(), jobHistoryFilePath); jobHistoryFilenameMap.put(job.getJobID(), jobHistoryFilePath); Path jobHistoryFilePath = jobHistoryFilenameMap.get(pJob.getJobID()); String logFileName = jobHistoryFilePath.getName(); pJob.getJobConf()); lastSeenEventMapOnRestart.put(pJob.getStatus().getJobID(), pJob.getNumTaskCompletionEvents());
return jobs.get(jobId).getStatus(); JobInProgress.copyJobFileLocally(jobDir, jobId, this.conf); job = new JobInProgress(jobId, this, jobTrackerReconfigurable.getJobConf(), ugi.getUserName(), 0); } catch (Exception e) { String queue = job.getProfile().getQueueName(); if (!(queueManager.getQueues().contains(queue))) { new CleanupQueue().addToQueue(new PathDeletionContext( FileSystem.get(conf), getSystemDirectoryForJob(jobId).toUri().getPath())); job.fail(); if (userFileForJob != null) { userFileForJob.delete(); checkAccess(job, QueueManager.QueueOperation.SUBMIT_JOB); } catch (IOException ioe) { LOG.warn("Access denied for user " + job.getJobConf().getUser() + ". Ignoring job " + jobId, ioe); job.fail(); if (userFileForJob != null) { userFileForJob.delete();
Meter meter = metrics.jobStateMeter.get(job.getStatus().getRunState()); if (meter != null) { meter.mark(); final List<String> flakyTrackers = job.getBlackListedTrackers(); if (!job.isComplete()) { return; com.codahale.metrics.Timer.Context context = metrics.jobTimerContexts.get(job.getJobID()); context.stop(); metrics.jobTimerContexts.remove(job.getJobID()); LOG.info("Completed job : " + job.getJobID()); for (HttpHost tracker : trackers) { MesosTracker mesosTracker = mesosTrackers.get(tracker); mesosTracker.jobs.remove(job.getJobID());
if (tasksInited.get() || isComplete()) { return; JobHistory.JobInfo.logSubmitted(getJobID(), conf, jobFile.toString(), this.startTime, hasRestarted()); setPriority(this.priority); " exceeds the configured limit " + maxTasks); jobtracker.getInstrumentation().addWaitingMaps(getJobID(), numMapTasks); jobtracker.getInstrumentation().addWaitingReduces(getJobID(), numReduceTasks); + ". Number of splits = " + splits.length); if (numMapTasks > 0) { nonRunningMapCache = createCache(splits, maxLevel); MAX_FETCH_FAILURES_PER_MAP_DEFAULT); initSetupCleanupTasks(jobFile); refreshIfNecessary();
private synchronized void killJob(JobInProgress job) { LOG.info("Killing job " + job.getJobID()); JobStatus prevStatus = (JobStatus)job.getStatus().clone(); job.kill(); // Inform the listeners if the job is killed // Note : // If the job is killed in the PREP state then the listeners will be // invoked // If the job is killed in the RUNNING state then cleanup tasks will be // launched and the updateTaskStatuses() will take care of it JobStatus newStatus = (JobStatus)job.getStatus().clone(); if (prevStatus.getRunState() != newStatus.getRunState() && newStatus.getRunState() == JobStatus.KILLED) { JobStatusChangeEvent event = new JobStatusChangeEvent(job, EventType.RUN_STATE_CHANGED, prevStatus, newStatus); updateJobInProgressListeners(event); } }