/** * Method for calculating Job weight by total task count. If * the total task count is greater than the size of the tasks * cache, then cap it at the cache size. This allows the cache * to always hold one large job. * @param key JobId object * @param value Job object * @return Weight of the job as calculated by total task count */ @Override public int weigh(JobId key, Job value) { int taskCount = Math.min(loadedTasksCacheSize, value.getTotalMaps() + value.getTotalReduces()); return taskCount; } };
/** * Method for calculating Job weight by total task count. If * the total task count is greater than the size of the tasks * cache, then cap it at the cache size. This allows the cache * to always hold one large job. * @param key JobId object * @param value Job object * @return Weight of the job as calculated by total task count */ @Override public int weigh(JobId key, Job value) { int taskCount = Math.min(loadedTasksCacheSize, value.getTotalMaps() + value.getTotalReduces()); return taskCount; } };
/** * Method for calculating Job weight by total task count. If * the total task count is greater than the size of the tasks * cache, then cap it at the cache size. This allows the cache * to always hold one large job. * @param key JobId object * @param value Job object * @return Weight of the job as calculated by total task count */ @Override public int weigh(JobId key, Job value) { int taskCount = Math.min(loadedTasksCacheSize, value.getTotalMaps() + value.getTotalReduces()); return taskCount; } };
private void applyConcurrentTaskLimits() { int numScheduledMaps = scheduledRequests.maps.size(); if (maxRunningMaps > 0 && numScheduledMaps > 0 && getJob().getTotalMaps() > maxRunningMaps) { int maxRequestedMaps = Math.max(0, maxRunningMaps - assignedRequests.maps.size()); int numScheduledFailMaps = scheduledRequests.earlierFailedMaps.size(); int failedMapRequestLimit = Math.min(maxRequestedMaps, numScheduledFailMaps); int normalMapRequestLimit = Math.min( maxRequestedMaps - failedMapRequestLimit, numScheduledMaps - numScheduledFailMaps); setRequestLimit(PRIORITY_FAST_FAIL_MAP, mapResourceRequest, failedMapRequestLimit); setRequestLimit(PRIORITY_MAP, mapResourceRequest, normalMapRequestLimit); setRequestLimit(PRIORITY_OPPORTUNISTIC_MAP, mapResourceRequest, normalMapRequestLimit); } int numScheduledReduces = scheduledRequests.reduces.size(); if (maxRunningReduces > 0 && numScheduledReduces > 0 && getJob().getTotalReduces() > maxRunningReduces) { int maxRequestedReduces = Math.max(0, maxRunningReduces - assignedRequests.reduces.size()); int reduceRequestLimit = Math.min(maxRequestedReduces, numScheduledReduces); setRequestLimit(PRIORITY_REDUCE, reduceResourceRequest, reduceRequestLimit); } }
private Job mockJob() { Job mockJob = mock(Job.class); when(mockJob.getAllCounters()).thenReturn(new Counters()); when(mockJob.getTotalMaps()).thenReturn(10); when(mockJob.getTotalReduces()).thenReturn(10); when(mockJob.getName()).thenReturn("mockjob"); return mockJob; }
this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.submitTime = report.getSubmitTime();
this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.submitTime = report.getSubmitTime();
this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.submitTime = report.getSubmitTime();
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
@Override public long thresholdRuntime(TaskId taskID) { JobId jobID = taskID.getJobId(); Job job = context.getJob(jobID); TaskType type = taskID.getTaskType(); DataStatistics statistics = dataStatisticsForTask(taskID); int completedTasksOfType = type == TaskType.MAP ? job.getCompletedMaps() : job.getCompletedReduces(); int totalTasksOfType = type == TaskType.MAP ? job.getTotalMaps() : job.getTotalReduces(); if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE || (((float)completedTasksOfType) / totalTasksOfType) < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) { return Long.MAX_VALUE; } long result = statistics == null ? Long.MAX_VALUE : (long)statistics.outlier(slowTaskRelativeTresholds.get(job)); return result; }
this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent = percent(report.getMapProgress()); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100;
this.mapProgressPercent = StringUtils.format("%.2f", getMapProgress()); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100;
this.mapProgressPercent = StringUtils.format("%.2f", getMapProgress()); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100;
0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); doReturn(10).when(job).getTotalMaps(); doReturn(10).when(job).getTotalReduces(); doReturn(0).when(job).getCompletedMaps(); RecalculateContainerAllocator allocator =
0, 0, 0, 0, 0, 0, "jobfile", null, false, "")); when(mockJob.getTotalMaps()).thenReturn(MAP_COUNT); when(mockJob.getTotalReduces()).thenReturn(REDUCE_COUNT);
int numReduceTasks = job.getTotalReduces();
int numReduceTasks = job.getTotalReduces();
public void verifyAMJobGeneric(Job job, String id, String user, String name, String state, long startTime, long finishTime, long elapsedTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted, float reduceProgress, float mapProgress) { JobReport report = job.getReport(); WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), id); WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(), user); WebServicesTestUtils.checkStringMatch("name", job.getName(), name); WebServicesTestUtils.checkStringMatch("state", job.getState().toString(), state); assertEquals("startTime incorrect", report.getStartTime(), startTime); assertEquals("finishTime incorrect", report.getFinishTime(), finishTime); assertEquals("elapsedTime incorrect", Times.elapsed(report.getStartTime(), report.getFinishTime()), elapsedTime); assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal); assertEquals("mapsCompleted incorrect", job.getCompletedMaps(), mapsCompleted); assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal); assertEquals("reducesCompleted incorrect", job.getCompletedReduces(), reducesCompleted); assertEquals("mapProgress incorrect", report.getMapProgress() * 100, mapProgress, 0); assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100, reduceProgress, 0); }
summary.setJobStatus(juce.getStatus()); summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps()); summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces()); summary.setJobFinishTime(juce.getFinishTime()); setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters());