@Test public void testCircularDependency() throws IOException { ControlledJob job1 = new ControlledJob(new Configuration()); job1.setJobName("job1"); ControlledJob job2 = new ControlledJob(new Configuration()); job2.setJobName("job2"); ControlledJob job3 = new ControlledJob(new Configuration()); job3.setJobName("job3"); job1.addDependingJob(job2); job2.addDependingJob(job3); job3.addDependingJob(job1); JobControl jobControl = new JobControl("test"); jobControl.addJob(job1); jobControl.addJob(job2); jobControl.addJob(job3); try { jobControl.run(); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); } } }
/** * @return the jobs in the success state */ public ArrayList<Job> getSuccessfulJobs() { return castToJobList(super.getSuccessfulJobList()); }
/** * @return the jobs in the running state */ public ArrayList<Job> getRunningJobs() { return castToJobList(super.getRunningJobList()); }
/** * Add a new job. * @param aJob the new job */ synchronized public String addJob(ControlledJob aJob) { String id = this.getNextJobID(); aJob.setJobID(id); aJob.setJobState(State.WAITING); this.addToQueue(aJob); return id; }
@Test public void testSuccessfulJobs() throws Exception { JobControl jobControl = new JobControl("Test"); ControlledJob job1 = createSuccessfulControlledJob(jobControl); ControlledJob job2 = createSuccessfulControlledJob(jobControl); ControlledJob job3 = createSuccessfulControlledJob(jobControl, job1, job2); ControlledJob job4 = createSuccessfulControlledJob(jobControl, job3); runJobControl(jobControl); assertEquals("Success list", 4, jobControl.getSuccessfulJobList().size()); assertEquals("Failed list", 0, jobControl.getFailedJobList().size()); assertTrue(job1.getJobState() == ControlledJob.State.SUCCESS); assertTrue(job2.getJobState() == ControlledJob.State.SUCCESS); assertTrue(job3.getJobState() == ControlledJob.State.SUCCESS); assertTrue(job4.getJobState() == ControlledJob.State.SUCCESS); jobControl.stop(); }
public Double call() throws Exception JobControl jobControl = new JobControl("YDiff job"); jobControl.addJob(controlledJob); Thread thread = new Thread(jobControl); thread.start(); while(!jobControl.allFinished()) jobControl.stop(); FileSystem fs = FileSystem.get(job.getConfiguration()); FSDataInputStream in =fs.open(new Path(outputFilePath+"/part-r-00000"));
List<ControlledJob> jobsWithoutIds = this.jc.getWaitingJobList(); int totalNeededMRJobs = jobsWithoutIds.size(); LOG.info("{} map-reduce job(s) waiting for submission.", jobsWithoutIds.size()); while(!this.jc.allFinished()) { try { jcThread.join(1000); List<ControlledJob> runningJobs = jc.getRunningJobList(); for(ControlledJob controlledJob: runningJobs) { String jobId = controlledJob.getJob().getJobID().toString(); List<ControlledJob> successfulJobs = jc.getSuccessfulJobList(); for(ControlledJob controlledJob: successfulJobs) { String jobId = controlledJob.getJob().getJobID().toString(); List<ControlledJob> failedJobs = jc.getFailedJobList(); for(ControlledJob controlledJob: failedJobs) { String failedJobId = controlledJob.getJob().getJobID().toString(); LOG.warn("Job {} is failed, will be submitted again.", jobName); Job newJob = createJob(this.jobIndexParams.get(jobIndex)); this.jc.addJob(new ControlledJob(newJob, null)); this.jobRunningTimes.put(jobIndex, runTimes + 1); this.jobIndexMap.put(newJob.getJobName(), jobIndex); jobsWithoutIds = this.jc.getWaitingJobList(); } else { LOG.warn("Job {} is failed twice, will not be submitted again.", jobName);
JobControl jobControl = new JobControl("job_chain"); controlledJob.addDependingJob(jobPool.get(i-1)); jobControl.addJob(controlledJob); jobPool.add(controlledJob); while(!jobControl.allFinished()){ System.out.println("Jobs in waiting state: " + jobControl.getWaitingJobList().size()); System.out.println("Jobs in ready state: " + jobControl.getReadyJobsList().size()); System.out.println("Jobs in running state: " + jobControl.getRunningJobList().size()); System.out.println("Jobs in success state: " + jobControl.getSuccessfulJobList().size()); System.out.println("Jobs in failed state: " + jobControl.getFailedJobList().size()); System.out.println(); try{ System.out.println(jobControl.getFailedJobList()); return 0;
public static JobControl createValueAggregatorJobs(String args[], Class<? extends ValueAggregatorDescriptor>[] descriptors) throws IOException { JobControl theControl = new JobControl("ValueAggregatorJobs"); ArrayList<ControlledJob> dependingJobs = new ArrayList<ControlledJob>(); Configuration conf = new Configuration(); if (descriptors != null) { conf = setAggregatorDescriptors(descriptors); } Job job = createValueAggregatorJob(conf, args); ControlledJob cjob = new ControlledJob(job, dependingJobs); theControl.addJob(cjob); return theControl; }
/** * Add a new job. * @param aJob the new job */ synchronized public String addJob(Job aJob) { return addJob((ControlledJob) aJob); }
public Double call() throws NectarException JobControl jobControl = new JobControl("sigmajob"); try jobControl.addJob(controlledJob); Thread thread = new Thread(jobControl); thread.start(); while(!jobControl.allFinished())
public void testJobControl() throws Exception { Configuration conf = createJobConf(); cleanupData(conf); Job job1 = MapReduceTestUtil.createCopyJob(conf, outdir_1, indir); JobControl theControl = createDependencies(conf, job1); // wait till all the jobs complete waitTillAllFinished(theControl); assertEquals("Some jobs failed", 0, theControl.getFailedJobList().size()); theControl.stop(); } }
/** * Compute the progress of the current job submitted through the JobControl object jc to the JobClient jobClient * * @param jc * The JobControl object that has been submitted * @param jobClient * The JobClient to which it has been submitted * @return The progress as a percentage in double format * @throws IOException * In case any IOException connecting to JobTracker. */ protected double calculateProgress(Set<String> successJobs, JobControl jc, JobClient jobClient) throws IOException { double prog = 0.0; prog += Math.max(jc.getSuccessfulJobList().size(), successJobs.size()); List<ControlledJob> runnJobs = jc.getRunningJobList(); for(ControlledJob cjob: runnJobs) { prog += progressOfRunningJob(cjob, jobClient); } return prog; }
/** * Default constructor. Construct default JobControl instance. */ public GuaguaMapReduceClient() { this.jc = new JobControl(INIT_JOB_ID_PREFIX); }
theControl.suspend(); assertTrue( theControl.getThreadState() == JobControl.ThreadState.SUSPENDED); theControl.resume(); assertTrue(cjob4.getJobState() == ControlledJob.State.DEPENDENT_FAILED); theControl.stop();
public ArrayList<Job> getFailedJobs() { return castToJobList(super.getFailedJobList()); }
/** * @return the jobs in the waiting state */ public ArrayList<Job> getWaitingJobs() { return castToJobList(super.getWaitingJobList()); }
/** * @return the thread state */ public int getState() { ThreadState state = super.getThreadState(); if (state == ThreadState.RUNNING) { return 0; } if (state == ThreadState.SUSPENDED) { return 1; } if (state == ThreadState.STOPPED) { return 2; } if (state == ThreadState.STOPPING) { return 3; } if (state == ThreadState.READY ) { return 4; } return -1; }
/** * @return the jobs in the ready state */ public ArrayList<Job> getReadyJobs() { return castToJobList(super.getReadyJobsList()); }
private void waitTillAllFinished(JobControl jobControl) { while (!jobControl.allFinished()) { try { Thread.sleep(100); } catch (InterruptedException e) { // ignore } } } }