private RunningJob addTaskToJob(JobID jobId, TaskInProgress tip) { synchronized (runningJobs) { RunningJob rJob = null; if (!runningJobs.containsKey(jobId)) { rJob = new RunningJob(jobId); rJob.localized = false; rJob.tasks = new HashSet<TaskInProgress>(); runningJobs.put(jobId, rJob); } else { rJob = runningJobs.get(jobId); } synchronized (rJob) { rJob.tasks.add(tip); } runningJobs.notify(); //notify the fetcher thread return rJob; } }
if (rjob != null) { synchronized (rjob) { FetchStatus f = rjob.getFetchStatus(); if (f != null) { f.purgeMapEvents(entry.getValue());
FetchStatus f; synchronized (rjob) { f = rjob.getFetchStatus(); for (TaskInProgress tip : rjob.tasks) { Task task = tip.getTask(); if (((ReduceTask)task).getPhase() == TaskStatus.Phase.SHUFFLE) { if (rjob.getFetchStatus() == null) { rjob.setFetchStatus(f); f = rjob.getFetchStatus(); fList.add(f); break; //no need to check any more tasks belonging to this
localizedJobConf = rjob.getJobConf();
getLocalFiles(fConf, getLocalJobDir(rjob.getJobID().toString()))); directoryCleanupThread.addToQueue(contexts);
localizedJobConf = rjob.getJobConf(); initializeTask(); localizedJobConf = rjob.getJobConf(); checkJobLocalization();
localizedJobConf = rjob.getJobConf(); initializeTask(); localizedJobConf = rjob.getJobConf(); checkJobLocalization();
getLocalJobDir(rjob.getJobID().toString())));
FetchStatus f; synchronized (rjob) { f = rjob.getFetchStatus(); for (TaskInProgress tip : rjob.tasks) { Task task = tip.getTask(); if (((ReduceTask)task).getPhase() == TaskStatus.Phase.SHUFFLE) { if (rjob.getFetchStatus() == null) { rjob.setFetchStatus(f); f = rjob.getFetchStatus(); fList.add(f); break; //no need to check any more tasks belonging to this
FetchStatus f; synchronized (rjob) { f = rjob.getFetchStatus(); for (TaskInProgress tip : rjob.tasks) { Task task = tip.getTask(); if (((ReduceTask)task).getPhase() == TaskStatus.Phase.SHUFFLE) { if (rjob.getFetchStatus() == null) { rjob.setFetchStatus(f); f = rjob.getFetchStatus(); fList.add(f); break; //no need to check any more tasks belonging to this
public boolean fetchMapCompletionEvents(long currTime) throws IOException { if (!fetchAgain && (currTime - lastFetchTime) < heartbeatInterval) { return false; } int currFromEventId = 0; synchronized (fromEventId) { currFromEventId = fromEventId.get(); List <TaskCompletionEvent> recentMapEvents = queryJobTracker(fromEventId, jobId, rJob.getJobClient()); synchronized (allMapEvents) { allMapEvents.addAll(recentMapEvents); } lastFetchTime = currTime; if (fromEventId.get() - currFromEventId >= probe_sample_size) { //return true when we have fetched the full payload, indicating //that we should fetch again immediately (there might be more to //fetch fetchAgain = true; return true; } } fetchAgain = false; return false; } }
public synchronized MapTaskCompletionEventsUpdate getMapCompletionEvents( JobID jobId, int fromEventId, int maxLocs, TaskAttemptID id) throws IOException { TaskCompletionEvent[]mapEvents = TaskCompletionEvent.EMPTY_ARRAY; synchronized (shouldReset) { if (shouldReset.remove(id)) { return new MapTaskCompletionEventsUpdate(mapEvents, true); } } RunningJob rjob; synchronized (runningJobs) { rjob = runningJobs.get(jobId); if (rjob != null) { synchronized (rjob) { FetchStatus f = rjob.getFetchStatus(); if (f != null) { mapEvents = f.getMapEvents(fromEventId, maxLocs); } } } } return new MapTaskCompletionEventsUpdate(mapEvents, false); }
/** * Localizes a cleanup task and validates permissions. * * @throws InterruptedException * @throws IOException */ public void testCleanupTaskLocalization() throws IOException, InterruptedException { if (!canRun()) { return; } task.setTaskCleanupTask(); // register task tip = tracker.new TaskInProgress(task, trackerFConf); // localize the job. RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); checkJobLocalization(); // localize task cleanup attempt initializeTask(); checkTaskLocalization(); } }
public synchronized MapTaskCompletionEventsUpdate getMapCompletionEvents( JobID jobId, int fromEventId, int maxLocs, TaskAttemptID id) throws IOException { TaskCompletionEvent[]mapEvents = TaskCompletionEvent.EMPTY_ARRAY; synchronized (shouldReset) { if (shouldReset.remove(id)) { return new MapTaskCompletionEventsUpdate(mapEvents, true); } } RunningJob rjob; synchronized (runningJobs) { rjob = runningJobs.get(jobId); if (rjob != null) { synchronized (rjob) { FetchStatus f = rjob.getFetchStatus(); if (f != null) { mapEvents = f.getMapEvents(fromEventId, maxLocs); } } } } return new MapTaskCompletionEventsUpdate(mapEvents, false); }
private RunningJob addTaskToJob(JobID jobId, TaskInProgress tip) { synchronized (runningJobs) { RunningJob rJob = null; if (!runningJobs.containsKey(jobId)) { rJob = new RunningJob(jobId); rJob.localized = false; rJob.tasks = new HashSet<TaskInProgress>(); runningJobs.put(jobId, rJob); } else { rJob = runningJobs.get(jobId); } synchronized (rJob) { rJob.tasks.add(tip); } runningJobs.notify(); //notify the fetcher thread return rJob; } }
public synchronized MapTaskCompletionEventsUpdate getMapCompletionEvents( JobID jobId, int fromEventId, int maxLocs, TaskAttemptID id) throws IOException { TaskCompletionEvent[]mapEvents = TaskCompletionEvent.EMPTY_ARRAY; RunningJob rjob; synchronized (runningJobs) { rjob = runningJobs.get(jobId); if (rjob != null) { synchronized (rjob) { FetchStatus f = rjob.getFetchStatus(); if (f != null) { mapEvents = f.getMapEvents(fromEventId, maxLocs); } } } } return new MapTaskCompletionEventsUpdate(mapEvents, false); }
/** * Test task localization on a TT. * * @throws IOException */ public void testTaskLocalization() throws Exception { if (!canRun()) { return; } TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); initializeTask(); checkTaskLocalization(); }
/** * Test job localization on a TT. Tests localization of job.xml, job.jar and * corresponding setting of configuration. Also test * {@link TaskController#initializeJob(JobInitializationContext)} * * @throws IOException */ public void testJobLocalization() throws Exception { if (!canRun()) { return; } TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); checkJobLocalization(); }
/** * Validates if task cleanup is done properly */ private void testTaskFilesRemoval(boolean needCleanup, boolean jvmReuse) throws Exception { // Localize job and localize task. TaskTracker.RunningJob rjob = tracker.localizeJob(tip); localizedJobConf = rjob.getJobConf(); if (jvmReuse) { localizedJobConf.setNumTasksToExecutePerJvm(2); } initializeTask(); // TODO: Let the task run and create files. // create files and set permissions 555. Verify if task controller sets // the permissions for TT to delete the task dir or work dir properly validateRemoveTaskFiles(needCleanup, jvmReuse, tip); }