private int getTrackerTaskFailures(String trackerName) { String trackerHostName = convertTrackerNameToHostName(trackerName); Integer failedTasks = trackerToFailuresMap.get(trackerHostName); return (failedTasks != null) ? failedTasks.intValue() : 0; }
private int getTrackerTaskFailures(String trackerName) { String trackerHostName = convertTrackerNameToHostName(trackerName); List<String> failedTasks = trackerToFailuresMap.get(trackerHostName); return (failedTasks != null) ? failedTasks.size() : 0; }
private int getTrackerTaskFailures(String trackerName) { String trackerHostName = convertTrackerNameToHostName(trackerName); Integer failedTasks = trackerToFailuresMap.get(trackerHostName); return (failedTasks != null) ? failedTasks.intValue() : 0; }
TaskTrackerStatus createTaskTrackerStatus(String tracker, List<TaskStatus> reports) { TaskTrackerStatus ttStatus = new TaskTrackerStatus(tracker, JobInProgress.convertTrackerNameToHostName(tracker), 0, reports, 0, 2, 2); return ttStatus; }
public TaskAttemptID findMapTask(String trackerName) throws IOException { return findTask(trackerName, JobInProgress.convertTrackerNameToHostName(trackerName), nonLocalMaps, nonLocalRunningMaps, TaskType.MAP); }
private TaskTrackerStatus getTTStatus(String trackerName, List<TaskStatus> taskStatuses) { return new TaskTrackerStatus(trackerName, JobInProgress.convertTrackerNameToHostName(trackerName), 0, taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker); }
private TaskTrackerStatus getTTStatus(String trackerName, List<TaskStatus> taskStatuses) { return new TaskTrackerStatus(trackerName, JobInProgress.convertTrackerNameToHostName(trackerName), 0, taskStatuses, 0, mapSlotsPerTracker, reduceSlotsPerTracker); }
public TaskAttemptID findReduceTask(String trackerName) throws IOException { return findTask(trackerName, JobInProgress.convertTrackerNameToHostName(trackerName), nonRunningReduces, runningReduces, TaskType.REDUCE); }
private boolean shouldRunOnTaskTracker(String taskTracker) { // // Check if too many tasks of this job have failed on this // tasktracker prior to assigning it a new one. // int taskTrackerFailedTasks = getTrackerTaskFailures(taskTracker); if ((flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) && taskTrackerFailedTasks >= maxTaskFailuresPerTracker) { if (LOG.isDebugEnabled()) { String flakyTracker = convertTrackerNameToHostName(taskTracker); LOG.debug("Ignoring the black-listed tasktracker: '" + flakyTracker + "' for assigning a new task"); } return false; } return true; }
public List<String> getBlackListedTrackers() { ArrayList<String> blackListedTrackers = new ArrayList<String>(); for (Entry<String, Integer> entry : trackerToFailureMap.entrySet()) { Integer failures = entry.getValue(); String tracker = entry.getKey(); if (failures.intValue() >= this.getJobConf() .getMaxTaskFailuresPerTracker()) { blackListedTrackers.add(JobInProgress .convertTrackerNameToHostName(tracker)); } } return blackListedTrackers; } }
private boolean shouldRunOnTaskTracker(String taskTracker) { // // Check if too many tasks of this job have failed on this // tasktracker prior to assigning it a new one. // int taskTrackerFailedTasks = getTrackerTaskFailures(taskTracker); if ((flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) && taskTrackerFailedTasks >= maxTaskFailuresPerTracker) { if (LOG.isDebugEnabled()) { String flakyTracker = convertTrackerNameToHostName(taskTracker); if (LOG.isDebugEnabled()) { LOG.debug("Ignoring the black-listed tasktracker: '" + flakyTracker + "' for assigning a new task"); } } return false; } return true; }
private boolean shouldRunOnTaskTracker(String taskTracker) { // // Check if too many tasks of this job have failed on this // tasktracker prior to assigning it a new one. // int taskTrackerFailedTasks = getTrackerTaskFailures(taskTracker); if ((flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) && taskTrackerFailedTasks >= conf.getMaxTaskFailuresPerTracker()) { if (LOG.isDebugEnabled()) { String flakyTracker = convertTrackerNameToHostName(taskTracker); LOG.debug("Ignoring the black-listed tasktracker: '" + flakyTracker + "' for assigning a new task"); } return false; } return true; }
private void establishFirstContact(String tracker) throws IOException { TaskTrackerStatus status = new TaskTrackerStatus(tracker, JobInProgress.convertTrackerNameToHostName(tracker)); jobTracker.heartbeat(status, false, true, false, (short) 0); }
static short sendHeartBeat(JobTracker jt, TaskTrackerStatus status, boolean initialContact, boolean acceptNewTasks, String tracker, short responseId) throws IOException { if (status == null) { status = new TaskTrackerStatus(tracker, JobInProgress.convertTrackerNameToHostName(tracker)); } jt.heartbeat(status, false, initialContact, acceptNewTasks, responseId); return ++responseId ; }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(JobHistory.TaskAttempt attempt) { if (attempt.get(Keys.HTTP_PORT).equals("") || attempt.get(Keys.TRACKER_NAME).equals("") || attempt.get(Keys.TASK_ATTEMPT_ID).equals("")) { return null; } String taskTrackerName = JobInProgress.convertTrackerNameToHostName( attempt.get(Keys.TRACKER_NAME)).substring("tracker_".length()); return TaskLogServlet.getTaskLogUrl(taskTrackerName, attempt .get(Keys.HTTP_PORT), attempt.get(Keys.TASK_ATTEMPT_ID)); } }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(JobHistory.TaskAttempt attempt) { if (attempt.get(Keys.HTTP_PORT).equals("") || attempt.get(Keys.TRACKER_NAME).equals("") || attempt.get(Keys.TASK_ATTEMPT_ID).equals("")) { return null; } String taskTrackerName = JobInProgress.convertTrackerNameToHostName( attempt.get(Keys.TRACKER_NAME)); return TaskLogServlet.getTaskLogUrl(taskTrackerName, attempt .get(Keys.HTTP_PORT), attempt.get(Keys.TASK_ATTEMPT_ID)); } }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl( JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = JobInProgress.convertTrackerNameToHostName( attempt.getTrackerName()); return TaskLogServlet.getTaskLogUrl(taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
private void makeRunning(TaskAttemptID taskId, TaskInProgress tip, String taskTracker) { addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker, JobInProgress.convertTrackerNameToHostName(taskTracker)), true); TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, 0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker, tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters()); updateTaskStatus(tip, status); }
public void testReservedSlots() throws IOException { JobConf conf = new JobConf(); conf.setNumMapTasks(1); conf.setNumReduceTasks(1); conf.setSpeculativeExecution(false); //Set task tracker objects for reservation. TaskTracker tt2 = jobTracker.getTaskTracker(trackers[1]); TaskTrackerStatus status2 = new TaskTrackerStatus( trackers[1],JobInProgress.convertTrackerNameToHostName( trackers[1]),0,new ArrayList<TaskStatus>(), 0, 2, 2); tt2.setStatus(status2); fakeJob = new FakeJobInProgress(conf, jobTracker); fakeJob.setClusterSize(3); fakeJob.initTasks(); FakeObjectUtilities.sendHeartBeat(jobTracker, status2, false, true, trackers[1], responseId); responseId++; assertEquals("Mismtach in reserved map slots", numMapSlotsToReserve, mi.numReservedMapSlots); assertEquals("Mismtach in reserved red slots", numReduceSlotsToReserve, mi.numReservedReduceSlots); }
private static void sendHeartBeat(TaskTrackerHealthStatus status, boolean initialContact) throws IOException { for (String tracker : trackers) { TaskTrackerStatus tts = new TaskTrackerStatus(tracker, JobInProgress .convertTrackerNameToHostName(tracker)); if (status != null) { TaskTrackerHealthStatus healthStatus = tts.getHealthStatus(); healthStatus.setNodeHealthy(status.isNodeHealthy()); healthStatus.setHealthReport(status.getHealthReport()); healthStatus.setLastReported(status.getLastReported()); } jobTracker.heartbeat(tts, false, initialContact, false, responseId); } responseId++; }