public void doRun() { DateTime now = Utils.getCurrentTime(); List<JobSpec> jobs = dao.getJobs(); for (JobSpec aJob : jobs) { if (shouldJobRun(aJob, now)) { LOG.info("Adding job to queue:" + aJob); dao.addToQueue(new PlannedJob(aJob, now)); } } try { LOG.info(String.format("Sleeping for %d seconds...", (SLEEP_FOR / 1000))); Thread.sleep(SLEEP_FOR); } catch (InterruptedException e) { e.printStackTrace(); } }
public void run() { for (int i = 0; i < size; i++) { JobSpec job = TestAgent.getTestJob(UUID.randomUUID().toString(), dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } } };
@Before public void setUp() throws Exception { dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 1, 1); consumer.SLEEP_FOR = 1; AgentConsumer.setShouldSendErrorReports(false); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 0, 1); consumer.writeReportToLocal(folder.getRoot().getPath()); consumer.SLEEP_FOR = 20; AgentConsumer.setShouldSendErrorReports(false); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); agentDriver = getMockedDriver(dao, reporting); agentDriver.SLEEP_FOR = 10; consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 60, 1); AgentConsumer.setShouldSendErrorReports(false); consumer.SLEEP_FOR = 1; }
@Test(timeout=2000) public void testScriptReplace() { JobSpec aJob = getTestScript("Doug Lea", dao); String strFormat = "YYYYMMdd"; DateTimeFormatter format = QueryReplaceUtil.makeDateTimeFormat(strFormat); DateTime now = Utils.getCurrentTime(); String command = String.format("echo '${%s-1D} ${%s}' >&2; exit 1;", strFormat, strFormat); String error = String.format("%s %s", format.print(now.minusDays(1)), format.print(now)); aJob.setCode(command); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals(false, actual.isSuccess()); assertEquals(true, actual.isFailed()); String expected = CallableScript.genErrorMessage(aJob, error+"\n"); assertEquals(expected, actual.getExceptionMessage().get()); }
@Test public void testQueueLarge() { int count = 1000; for (int i = 1; i <= count; i++) { JobSpec aJob = TestAgent.getTestJob("Vincent Van Gogh", dao); long id = dao.createJob(aJob); aJob = dao.getJob(id); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); dao.addToQueue(pj); } assertEquals(count, dao.getQueue(null).size()); }
@Test public void testGetJobRuns() { List<Long> ids = new ArrayList<>(); Map<Long, CallableJob> expected = new HashMap<>(); for (String name : new String[] { "Wifredo Lam", "Rene Magritte" }) { JobSpec aJob = TestAgent.getTestJob(name, dao); try { dao.createJob(aJob); aJob = dao.getJob(aJob.getId()); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); dao.createJobRun(cj); ids.add(aJob.getId()); expected.put(aJob.getId(), cj); } catch (Exception ex) { ex.printStackTrace(); } } for (Long id : ids) { Map<Long, CallableJob> got = dao.getJobRuns(id, limit); assertEquals(1, got.size()); assertEquals(expected.get(id), got.get(id)); } Map<Long, CallableJob> got = dao.getJobRuns(null, limit); assertEquals(expected, got); }
public static JobSpec getTestJob(String aName, JobDao dao) { JobSpec aJob = new JobSpec(); aJob.setName(aName); DateTime now = Utils.getCurrentTime(); aJob.setCronString(String.format("%d * * * *", now.getHourOfDay())); aJob.setDriver(H2TestUtil.H2_NAME); aJob.setCode("show tables;"); aJob.setResultTable("ARESULTTABLE"); aJob.setEnabled(true); aJob.setStatusEmail(Collections.singletonList("blah@example.com")); aJob.setType(JobType.Query); return aJob; }
@Test public void testQueueJob() throws Exception { JobSpec job = TestAgent.getTestJob("blah", dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); List<PlannedJob> expected = new ArrayList<>(); expected.add(aJob); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } List<PlannedJob> actual = dao.getQueue(aJob.getJobSpec().getId()); assertEquals(expected, actual); PlannedJob actualPJ = null; try { actualPJ = dao.removeFromQueue(); } catch (Exception ex) { ex.printStackTrace(); } assertEquals(aJob, actualPJ); expected = new ArrayList<>(); actual = dao.getQueue(null); assertEquals(expected, actual); }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
@Test(timeout=2000) public void testBasic() throws Exception { String jobName = "Jean Paul Sartre"; for (JobSpec aJob : new JobSpec[]{ getTestJob(jobName, dao), getTestScript(jobName, dao) }) { aJob.setResultTable(null); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); dao.deleteJob(aJob.getId()); } }
@Test public void testCleanupPreviouslyRunningJobs() { JobSpec aJob = TestAgent.getTestJob("Mary Wollstonecraft", dao); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); SleepyCallableQuery cj = new SleepyCallableQuery( pj, dao, reporting, "example.com", mailInfo, null, null, drivers.get(0), 1, 10000); consumer.submitJob(cj); doSleep(); assertEquals(0, consumer.getFailedQueries(limit).size()); assertEquals(1, dao.getRunningJobs().size()); AgentConsumer.cleanupPreviouslyRunningJobs(dao, dao.getRunningJobs()); assertEquals(0, dao.getRunningJobs().size()); assertEquals(1, consumer.getFailedQueries(limit).size()); }
@Test(timeout=2000) public void testBasicWithFail() throws Exception { String resultTable = "SHOULDNT_EXIST"; // make sure table doesn't exist dropTable(resultTable); JobSpec aJob = getTestJob("Michael Scott", dao); aJob.setResultTable(resultTable); // Second query should not be executed since the first one fails aJob.setCode("select * from something_that_fails; " + String.format("CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); List<String> actual = getResults(aJob); assertEquals(new ArrayList<String>(), actual); assertEquals(false, dao.showTables().contains(resultTable)); }
@Test public void testReplacedReportQuery() { JobSpec aJob = TestAgent.getTestJob("Mark Rothko", dao); aJob.setResultQuery("select * from blah where dt=${YYYMMDDHH}"); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); String expected = "select * from blah where dt=1970010100"; Assert.assertEquals(expected, cq.getReplacedReportQuery()); }
@Test public void testGetFailedQueries() { JobSpec aJob = TestAgent.getTestJob("David Foster Wallace", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); assertEquals(true, cj.isDone()); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals(1, consumer.getFinishedJobs(limit).size()); }
@Test public void testJobNoResubmit() throws BackendException { consumer.SLEEP_FOR = 1; JobSpec aJob = TestAgent.getTestJob("Hannah Arendt", dao); aJob.setCode("not a valid query..."); aJob.setShouldRerun(false); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); // let the job run and fail TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); // See if the job retries TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 1, jobRuns.values().size()); assertEquals(1, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); }
@Test public void testCancelPendingJob() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Foucault", dao); aJob.setShouldRerun(true); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); dao.addToQueue(pj); int actual = dao.cancelJob(pj); assertEquals(1, actual); actual = dao.cancelJob(pj); assertEquals(0, actual); TestAgent.runRunnable(consumer); assertEquals(0, dao.getRunningJobs().size()); assertEquals(0, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFailedQueries(limit).size()); }
@Test public void testJobResubmitSuccessUpdateQuery() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Robert Frank", dao); String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); Long nextId = cj.getJobId()+1; aJob = dao.getJob(aJob.getId()); aJob.setCode("show tables;"); dao.updateJob(aJob); TestAgent.runRunnable(consumer); doSleep(); assertEquals(0, dao.getRunningJobs().size()); assertEquals(2, consumer.getFinishedJobs(limit).size()); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); } @Test