public ResourceEstimator(JobInProgress job) { this.job = job; threshholdToUse = job.desiredMaps()/ 10; mapBlowupRatio = 0; estimateWeight = 1; }
public ResourceEstimator(JobInProgress job) { this.job = job; threshholdToUse = job.desiredMaps()/ 10; }
public ResourceEstimator(JobInProgress job) { this.job = job; threshholdToUse = job.desiredMaps()/ 10; }
/** * @return estimated length of this job's average map output */ long getEstimatedMapOutputSize() { long estimate = 0L; if (job.desiredMaps() > 0) { estimate = getEstimatedTotalMapOutputSize() / job.desiredMaps(); } return estimate; }
/** * @return estimated length of this job's average map output */ long getEstimatedMapOutputSize() { long estimate = 0L; if (job.desiredMaps() > 0) { estimate = getEstimatedTotalMapOutputSize() / job.desiredMaps(); } return estimate; }
@Override public boolean shouldSpeculateAllRemainingMaps() { if (speculativeMapUnfininshedThreshold == 0) { return false; } int unfinished = desiredMaps() - finishedMaps(); if (unfinished < desiredMaps() * speculativeMapUnfininshedThreshold || unfinished == 1) { return true; } return false; }
/** * @return estimated length of this job's average map output */ long getEstimatedMapOutputSize() { long estimate = getEstimatedTotalMapOutputSize() / job.desiredMaps(); return estimate; }
/** * @return estimated length of this job's total map output */ protected synchronized long getEstimatedTotalMapOutputSize() { if(completedMapsUpdates < threshholdToUse) { return 0; } else { long inputSize = job.getInputLength() + job.desiredMaps(); //add desiredMaps() so that randomwriter case doesn't blow up long estimate = Math.round((inputSize * completedMapsOutputSize * 2.0)/completedMapsInputSize); if (LOG.isDebugEnabled()) { LOG.debug("estimate total map output will be " + estimate); } return estimate; } }
/** * @return estimated length of this job's total map output */ protected synchronized long getEstimatedTotalMapOutputSize() { if(completedMapsUpdates < threshholdToUse) { return 0; } else { long inputSize = job.getInputLength() + job.desiredMaps(); //add desiredMaps() so that randomwriter case doesn't blow up long estimate = Math.round((inputSize * completedMapsOutputSize * 2.0)/completedMapsInputSize); if (LOG.isDebugEnabled()) { LOG.debug("estimate total map output will be " + estimate); } return estimate; } }
/** * @return estimated length of this job's total map output */ protected long getEstimatedTotalMapOutputSize() { double estWeight; synchronized(this) { estWeight = this.estimateWeight; } if(estWeight < threshholdToUse) { return 0; } else { double blowup =getBlowupRatio(); long inputSize = job.getInputLength() + job.desiredMaps(); //add desiredMaps() so that randomwriter case doesn't blow up long estimate = Math.round(inputSize * blowup * 2.0); LOG.debug("estimate total map output will be " + estimate + " bytes. (blowup = 2*" + blowup + ")"); return estimate; } }
JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps();
for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() == JobStatus.RUNNING) { neededMaps += job.desiredMaps() - job.finishedMaps(); neededReduces += job.desiredReduces() - job.finishedReduces();
for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() == JobStatus.RUNNING) { neededMaps += job.desiredMaps() - job.finishedMaps(); neededReduces += job.desiredReduces() - job.finishedReduces();
for (JobInProgress job : jobQueue) { if (job.getStatus().getRunState() == JobStatus.RUNNING) { neededMaps += job.desiredMaps() - job.finishedMaps(); neededReduces += job.desiredReduces() - job.finishedReduces();
private TaskInProgress createAndAddTIP(JobTracker jobtracker, JobInProgress jip, TaskType type) { JobConf conf = jip.getJobConf(); JobID id = jip.getJobID(); // now create a fake tip for this fake job TaskInProgress tip = null; if (type == TaskType.MAP) { tip = new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, jobtracker, conf, jip, 0, 1); jip.maps = new TaskInProgress[] {tip}; } else if (type == TaskType.REDUCE) { tip = new TaskInProgress(id, "dummy", jip.desiredMaps(), 0, jobtracker, conf, jip, 1); jip.reduces = new TaskInProgress[] {tip}; } else if (type == TaskType.JOB_SETUP) { tip = new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, jobtracker, conf, jip, 0, 1); jip.setup = new TaskInProgress[] {tip}; } else if (type == TaskType.JOB_CLEANUP) { tip = new TaskInProgress(id, "dummy", JobSplit.EMPTY_TASK_SPLIT, jobtracker, conf, jip, 0, 1); jip.cleanup = new TaskInProgress[] {tip}; } return tip; }
isMapTask ? job.desiredMaps() : job.desiredReduces(); int padding = 0; if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
isMapTask ? job.desiredMaps() : job.desiredReduces(); int padding = 0; if (numTaskTrackers > MIN_CLUSTER_SIZE_FOR_PADDING) {
/** * Returns an XML-formatted table of the jobs in the list. * This is called repeatedly for different lists of jobs (e.g., running, completed, failed). */ public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs) throws IOException { if (jobs.size() > 0) { for (JobInProgress job : jobs) { JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String name = profile.getJobName(); out.print("<" + label + "_job jobid=\"" + jobid + "\">\n"); out.print(" <jobid>" + jobid + "</jobid>\n"); out.print(" <user>" + profile.getUser() + "</user>\n"); out.print(" <name>" + ("".equals(name) ? " " : name) + "</name>\n"); out.print(" <map_complete>" + StringUtils.formatPercent(status.mapProgress(), 2) + "</map_complete>\n"); out.print(" <map_total>" + desiredMaps + "</map_total>\n"); out.print(" <maps_completed>" + completedMaps + "</maps_completed>\n"); out.print(" <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(), 2) + "</reduce_complete>\n"); out.print(" <reduce_total>" + desiredReduces + "</reduce_total>\n"); out.print(" <reduces_completed>" + completedReduces + "</reduces_completed>\n"); out.print("</" + label + "_job>\n"); } } }
/** * Returns an XML-formatted table of the jobs in the list. * This is called repeatedly for different lists of jobs (e.g., running, completed, failed). */ public void generateJobTable(JspWriter out, String label, List<JobInProgress> jobs) throws IOException { if (jobs.size() > 0) { for (JobInProgress job : jobs) { JobProfile profile = job.getProfile(); JobStatus status = job.getStatus(); JobID jobid = profile.getJobID(); int desiredMaps = job.desiredMaps(); int desiredReduces = job.desiredReduces(); int completedMaps = job.finishedMaps(); int completedReduces = job.finishedReduces(); String name = profile.getJobName(); out.print("<" + label + "_job jobid=\"" + jobid + "\">\n"); out.print(" <jobid>" + jobid + "</jobid>\n"); out.print(" <user>" + profile.getUser() + "</user>\n"); out.print(" <name>" + ("".equals(name) ? " " : name) + "</name>\n"); out.print(" <map_complete>" + StringUtils.formatPercent(status.mapProgress(), 2) + "</map_complete>\n"); out.print(" <map_total>" + desiredMaps + "</map_total>\n"); out.print(" <maps_completed>" + completedMaps + "</maps_completed>\n"); out.print(" <reduce_complete>" + StringUtils.formatPercent(status.reduceProgress(), 2) + "</reduce_complete>\n"); out.print(" <reduce_total>" + desiredReduces + "</reduce_total>\n"); out.print(" <reduces_completed>" + completedReduces + "</reduces_completed>\n"); out.print("</" + label + "_job>\n"); } } }
jobInfo.getTotalMaps() == jip.desiredMaps());