public void doRun() { DateTime now = Utils.getCurrentTime(); List<JobSpec> jobs = dao.getJobs(); for (JobSpec aJob : jobs) { if (shouldJobRun(aJob, now)) { LOG.info("Adding job to queue:" + aJob); dao.addToQueue(new PlannedJob(aJob, now)); } } try { LOG.info(String.format("Sleeping for %d seconds...", (SLEEP_FOR / 1000))); Thread.sleep(SLEEP_FOR); } catch (InterruptedException e) { e.printStackTrace(); } }
public void run() { for (int i = 0; i < size; i++) { JobSpec job = TestAgent.getTestJob(UUID.randomUUID().toString(), dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } expected.add(aJob); } } };
@Before public void setUp() throws Exception { dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 1, 1); consumer.SLEEP_FOR = 1; AgentConsumer.setShouldSendErrorReports(false); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 0, 1); consumer.writeReportToLocal(folder.getRoot().getPath()); consumer.SLEEP_FOR = 20; AgentConsumer.setShouldSendErrorReports(false); }
@Before public void setUp() throws Exception { reporting = new NoReporting(); dao = new H2TestJobDaoImpl(); dao.setDrivers(drivers); dao.init(); PowerMockito.mockStatic(Utils.class); when(Utils.getCurrentTime()) .thenReturn(new DateTime(0).withZone(DateTimeZone.UTC)); agentDriver = getMockedDriver(dao, reporting); agentDriver.SLEEP_FOR = 10; consumer = new AgentConsumer(dao, reporting, "testing.huffpo.com", new MailInfo("", "", "", ""), Session.getDefaultInstance(new Properties()), drivers, numOfConcurrentJobs, numOfConcurrentReruns, maxReruns, 60, 1); AgentConsumer.setShouldSendErrorReports(false); consumer.SLEEP_FOR = 1; }
@Test(timeout=2000) public void testScriptReplace() { JobSpec aJob = getTestScript("Doug Lea", dao); String strFormat = "YYYYMMdd"; DateTimeFormatter format = QueryReplaceUtil.makeDateTimeFormat(strFormat); DateTime now = Utils.getCurrentTime(); String command = String.format("echo '${%s-1D} ${%s}' >&2; exit 1;", strFormat, strFormat); String error = String.format("%s %s", format.print(now.minusDays(1)), format.print(now)); aJob.setCode(command); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals(false, actual.isSuccess()); assertEquals(true, actual.isFailed()); String expected = CallableScript.genErrorMessage(aJob, error+"\n"); assertEquals(expected, actual.getExceptionMessage().get()); }
aJob.setResultQuery("select * from blah where dt=${YYYMMDDHH}"); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao, null, "example.com", null, null,
@Test public void testQueueLarge() { int count = 1000; for (int i = 1; i <= count; i++) { JobSpec aJob = TestAgent.getTestJob("Vincent Van Gogh", dao); long id = dao.createJob(aJob); aJob = dao.getJob(id); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); dao.addToQueue(pj); } assertEquals(count, dao.getQueue(null).size()); }
@Test public void testGetJobRuns() { List<Long> ids = new ArrayList<>(); Map<Long, CallableJob> expected = new HashMap<>(); for (String name : new String[] { "Wifredo Lam", "Rene Magritte" }) { JobSpec aJob = TestAgent.getTestJob(name, dao); try { dao.createJob(aJob); aJob = dao.getJob(aJob.getId()); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); dao.createJobRun(cj); ids.add(aJob.getId()); expected.put(aJob.getId(), cj); } catch (Exception ex) { ex.printStackTrace(); } } for (Long id : ids) { Map<Long, CallableJob> got = dao.getJobRuns(id, limit); assertEquals(1, got.size()); assertEquals(expected.get(id), got.get(id)); } Map<Long, CallableJob> got = dao.getJobRuns(null, limit); assertEquals(expected, got); }
public static JobSpec getTestJob(String aName, JobDao dao) { JobSpec aJob = new JobSpec(); aJob.setName(aName); DateTime now = Utils.getCurrentTime(); aJob.setCronString(String.format("%d * * * *", now.getHourOfDay())); aJob.setDriver(H2TestUtil.H2_NAME); aJob.setCode("show tables;"); aJob.setResultTable("ARESULTTABLE"); aJob.setEnabled(true); aJob.setStatusEmail(Collections.singletonList("blah@example.com")); aJob.setType(JobType.Query); return aJob; }
Utils.getCurrentTime()); assertEquals(false, actual); Utils.getCurrentTime()); assertEquals(true, actual); Utils.getCurrentTime().withDayOfMonth(1)); assertEquals(true, actual); actual = AgentDriver.shouldJobRun(aJob, Utils.getCurrentTime().withDayOfMonth(2)); assertEquals(false, actual); Utils.getCurrentTime()); assertEquals(true, actual); actual = AgentDriver.shouldJobRun(aJob, Utils.getCurrentTime().plusDays(1)); assertEquals(false, actual); Utils.getCurrentTime()); assertEquals(true, actual); actual = AgentDriver.shouldJobRun(aJob, Utils.getCurrentTime().plusHours(1)); assertEquals(false, actual); Utils.getCurrentTime()); assertEquals(true, actual); Utils.getCurrentTime().plusHours(1)); assertEquals(true, actual);
@Test public void testQueueJob() throws Exception { JobSpec job = TestAgent.getTestJob("blah", dao); long id = dao.createJob(job); job = dao.getJob(id); PlannedJob aJob = new PlannedJob(job, Utils.getCurrentTime()); List<PlannedJob> expected = new ArrayList<>(); expected.add(aJob); try { dao.addToQueue(aJob); } catch (Exception ex) { ex.printStackTrace(); } List<PlannedJob> actual = dao.getQueue(aJob.getJobSpec().getId()); assertEquals(expected, actual); PlannedJob actualPJ = null; try { actualPJ = dao.removeFromQueue(); } catch (Exception ex) { ex.printStackTrace(); } assertEquals(aJob, actualPJ); expected = new ArrayList<>(); actual = dao.getQueue(null); assertEquals(expected, actual); }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
@Test(timeout=2000) public void testBasic() throws Exception { String jobName = "Jean Paul Sartre"; for (JobSpec aJob : new JobSpec[]{ getTestJob(jobName, dao), getTestScript(jobName, dao) }) { aJob.setResultTable(null); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); dao.deleteJob(aJob.getId()); } }
@Test public void testCleanupPreviouslyRunningJobs() { JobSpec aJob = TestAgent.getTestJob("Mary Wollstonecraft", dao); long id = dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(id), Utils.getCurrentTime()); SleepyCallableQuery cj = new SleepyCallableQuery( pj, dao, reporting, "example.com", mailInfo, null, null, drivers.get(0), 1, 10000); consumer.submitJob(cj); doSleep(); assertEquals(0, consumer.getFailedQueries(limit).size()); assertEquals(1, dao.getRunningJobs().size()); AgentConsumer.cleanupPreviouslyRunningJobs(dao, dao.getRunningJobs()); assertEquals(0, dao.getRunningJobs().size()); assertEquals(1, consumer.getFailedQueries(limit).size()); }
@Test(timeout=2000) public void testBasicWithFail() throws Exception { String resultTable = "SHOULDNT_EXIST"; // make sure table doesn't exist dropTable(resultTable); JobSpec aJob = getTestJob("Michael Scott", dao); aJob.setResultTable(resultTable); // Second query should not be executed since the first one fails aJob.setCode("select * from something_that_fails; " + String.format("CREATE TABLE %s AS SELECT time, url, type FROM %s;", aJob.getResultTable(), H2TestJobDaoImpl.testTableName)); try { dao.createJob(aJob); } catch (Exception ex) { ex.printStackTrace(); } runRunnable(agentDriver); List<PlannedJob> expected = new ArrayList<>(); expected.add(new PlannedJob(aJob, Utils.getCurrentTime())); assertEquals(expected, dao.getQueue(aJob.getId())); expected.clear(); runRunnable(consumer); assertEquals(expected, dao.getQueue(null)); List<String> actual = getResults(aJob); assertEquals(new ArrayList<String>(), actual); assertEquals(false, dao.showTables().contains(resultTable)); }
aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime());
@Test public void testReplacedReportQuery() { JobSpec aJob = TestAgent.getTestJob("Mark Rothko", dao); aJob.setResultQuery("select * from blah where dt=${YYYMMDDHH}"); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); String expected = "select * from blah where dt=1970010100"; Assert.assertEquals(expected, cq.getReplacedReportQuery()); }
JobSpec aJob = TestAgent.getTestJob("Joblah " + String.valueOf(i), dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); String.valueOf(numOfConcurrentJobs+i), dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(dao.getJob(aJob.getId()), Utils.getCurrentTime()); dao.addToQueue(pj); totalJobs++;