@Test(timeout=2000) public void testJobRun() { JobSpec aJob = getTestJob("Testing Job Run", dao); aJob.setCode(String.format( "CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); dao.createJob(aJob); runRunnable(agentDriver); runRunnable(consumer); boolean allFuturesDone = false; while (!allFuturesDone) { allFuturesDone = areAllFuturesDone(consumer); Assert.assertEquals(name, rerun.getPlannedJob().getJobSpec().getName()); List<String> actual = getResults(aJob); List<String> expected = Arrays.asList("1374101000\thttp://huffingtonpost.com/\tping", "1374100900\thttp://huffingtonpost.com/\tclick", assertEquals(aJob, actualJob); dropTable(aJob.getResultTable());
@Test public void testScriptRun() { JobSpec aJob = getTestScript("Chenoweth", dao); aJob.setCode("echo 'hi'; echo 'bye';"); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals("", actual.getExceptionMessage().get()); assertEquals(true, actual.isSuccess()); }
@Test public void testBasicDependent() throws Exception { String jobName = "TE Lawrence"; JobSpec parent = getTestJob(jobName, dao); JobSpec child = getTestJob(jobName + " arabia", dao); child.setCronString(null); try { dao.createJob(parent); child.setParent(parent.getId()); long id = dao.createJob(child); } catch (Exception ex) { ex.printStackTrace(); } TestAgent.runRunnable(agentDriver); assertEquals(1, dao.getQueue(null).size()); runRunnable(consumer); Thread.sleep(100); // child job should have been queued assertEquals(1, dao.getQueue(null).size()); runRunnable(consumer); waitUntilJobsFinished(consumer, 2); assertEquals(2, consumer.getSuccesfulQueries(AgentConsumer.LIMIT_JOB_RUNS) .values().size()); } }
@Test(timeout=2000) public void testBasic() throws Exception { String jobName = "Jean Paul Sartre"; for (JobSpec aJob : new JobSpec[]{ getTestJob(jobName, dao), getTestScript(jobName, dao) }) { aJob.setResultTable(null); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); dao.deleteJob(aJob.getId()); } }
@Test(timeout=60000) public void testJobResubmitSuccessSecondTime() throws BackendException { JobSpec aJob = TestAgent.getTestJob("DFW", dao); String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); TestAgent.runRunnable(consumer); TestAgent.runRunnable(consumer); doSleep(); TestAgent.runRunnable(consumer); TestAgent.runRunnable(consumer); TestAgent.waitUntilJobsFinished(consumer, 2);
@Test public void testGetJobs() { List<JobSpec> expected = new ArrayList<>(); for (String aName : new String[]{ "Allen Ginsberg", "Louis Gluck"}) { JobSpec aJob = TestAgent.getTestJob(aName, dao); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } List<JobSpec> actual = dao.getJobs(); assertEquals(expected, actual); }
@Test public void testJobResubmitSuccessUpdateQuery() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Robert Frank", dao); String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); Long nextId = cj.getJobId()+1; aJob = dao.getJob(aJob.getId()); aJob.setCode("show tables;"); dao.updateJob(aJob); TestAgent.runRunnable(consumer); doSleep(); assertEquals(0, dao.getRunningJobs().size()); assertEquals(2, consumer.getFinishedJobs(limit).size()); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); } @Test
@Test public void testCancelPendingJob() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Foucault", dao); aJob.setShouldRerun(true); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); dao.addToQueue(pj); int actual = dao.cancelJob(pj); assertEquals(1, actual); actual = dao.cancelJob(pj); assertEquals(0, actual); TestAgent.runRunnable(consumer); assertEquals(0, dao.getRunningJobs().size()); assertEquals(0, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFailedQueries(limit).size()); }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
@Test public void testGetFailedQueries() { JobSpec aJob = TestAgent.getTestJob("David Foster Wallace", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); assertEquals(true, cj.isDone()); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals(1, consumer.getFinishedJobs(limit).size()); }
public void run() { for (int i = 0 ; i < 1000; i++) { TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); } } });
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); agentDriver = getMockedDriver(dao, reporting); agentDriver.SLEEP_FOR = 10; consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 60, 1); AgentConsumer.setShouldSendErrorReports(false); consumer.SLEEP_FOR = 1; }
@Test(timeout=2000) public void testBasicWithFail() throws Exception { String resultTable = "SHOULDNT_EXIST"; // make sure table doesn't exist dropTable(resultTable); JobSpec aJob = getTestJob("Michael Scott", dao); aJob.setResultTable(resultTable); // Second query should not be executed since the first one fails aJob.setCode("select * from something_that_fails; " + String.format("CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); List<String> actual = getResults(aJob); assertEquals(new ArrayList<String>(), actual); assertEquals(false, dao.showTables().contains(resultTable)); }
@Test(timeout=10000) public void testBasicLarge() throws Exception { int count = 100; for (int i = 1; i <= count; i++) { JobSpec aJob = getTestJob(String.valueOf(i), dao); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } } runRunnable(agentDriver); assertEquals(count, dao.getQueue(null).size()); waitUntilJobsFinished(consumer, count); assertEquals(0, dao.getQueue(null).size()); assertEquals(count, dao.getJobRuns(null, count).size()); assertEquals(0, consumer.getFailedQueries(count).size()); assertEquals(count, consumer.getFinishedJobs(count).size()); assertEquals(count, consumer.getSuccesfulQueries(count).size()); }
public void run() { for (int i = 0; i < size; i++) { JobSpec job = TestAgent.getTestJob(UUID.randomUUID().toString(), dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } } };
JobSpec aJob = TestAgent.getTestJob("Foucault "+i, dao); aJob.setShouldRerun(true); long id = dao.createJob(aJob); JobSpec aJob = TestAgent.getTestJob("Foucault "+(numOfConcurrentJobs), dao); aJob.setType(JobType.Script); aJob.setCode("sleep 1; echo \"done\";"); assertEquals(0, consumer.getFailedQueries(limit).size()); TestAgent.runRunnable(consumer); doSleep(); TestAgent.runRunnable(consumer);
@Test public void testJobResubmitMaxFailAttempts() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Simone de Beauvoir", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess);
public static void waitUntilJobsFinished(AgentConsumer c, int count) { while (c.getFinishedJobs(AgentConsumer.LIMIT_JOB_RUNS).size() != count) { runRunnable(c); try { Thread.sleep(100); } catch (Exception ignore) { } } }
@Test public void testJobNoResubmit() throws BackendException { consumer.SLEEP_FOR = 1; JobSpec aJob = TestAgent.getTestJob("Hannah Arendt", dao); aJob.setCode("not a valid query..."); aJob.setShouldRerun(false); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); // let the job run and fail TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); // See if the job retries TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 1, jobRuns.values().size()); assertEquals(1, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); }
@Test(timeout=2000) public void testScriptRunFail() { JobSpec aJob = getTestScript("Chenoweth", dao); String error = "error"; aJob.setCode(String.format("echo '%s' >&2; exit 1;", error)); dao.createJob(aJob); runRunnable(agentDriver); runRunnable(consumer); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals(false, actual.isSuccess()); assertEquals(true, actual.isFailed()); String expected = CallableScript.genErrorMessage(aJob, error+"\n"); assertEquals(expected, actual.getExceptionMessage().get()); }