public void updateJobRun(DateTime dt, CallableJob cj) throws BackendException { Connection conn = null; PreparedStatement stat = null; try { conn = newConnection(); stat = conn.prepareStatement( String.format("UPDATE %s SET dt = ?, callable_job = ?, name = ?, `code` = ?, status = ?, exception = ?," + " start = ?, finish = ? WHERE id = ?", jobRunTableName)); int i = 1; stat.setTimestamp(i++, new Timestamp(dt.getMillis())); stat.setString(i++, objToString(cj)); stat.setString(i++, cj.getPlannedJob().getJobSpec().getName()); stat.setString(i++, cj.getPlannedJob().getJobSpec().getCode()); stat.setInt(i++, cj.getStatus().get()); stat.setString(i++, cj.getExceptionMessage() != null ? cj.getExceptionMessage().get() : ""); stat.setTimestamp(i++, new Timestamp(cj.getStart().get())); stat.setTimestamp(i++, new Timestamp(cj.getFinish().get())); stat.setLong(i++, cj.getJobId()); int rows = stat.executeUpdate(); LOG.info(String.format("Rows updated: %d", rows)); } catch (SQLException | IOException ex) { throw new BackendException(ex); } finally { closeConnections(conn, stat); } }
@Override public Void call() throws Exception { begin(); try { callInternal(); } catch (Exception ex) { handleException(ex); } finally { end(); } return null; }
synchronized public Map<Long, CallableJob> getFinishedJobs(int limit) { Map<Long, CallableJob> toRet = new TreeMap<>(); Map<Long, CallableJob> runs = dao.getJobRuns(null, limit); for (Entry<Long, CallableJob> entry : runs.entrySet()) { Long key = entry.getKey(); CallableJob value = entry.getValue(); boolean isRunning = value.isRunning(); boolean isDone = value.isDone(); if (!isRunning && isDone) { toRet.put(key, value); } } return toRet; }
final PlannedJob pj = cj.getPlannedJob(); final String jobName = pj.getJobSpec().getName(); if (!pj.getJobSpec().getShouldRerun()) { final int attempt = latest.getAttemptNumber(); boolean notMaxed = attempt < maxReruns; boolean latestFailed = latest.isDone() && !latest.isRunning() && latest.isFailed(); if (latestFailed && notMaxed) { synchronized (pendingReruns) {
@Test(timeout=2000) public void testScriptReplace() { JobSpec aJob = getTestScript("Doug Lea", dao); String strFormat = "YYYYMMdd"; DateTimeFormatter format = QueryReplaceUtil.makeDateTimeFormat(strFormat); DateTime now = Utils.getCurrentTime(); String command = String.format("echo '${%s-1D} ${%s}' >&2; exit 1;", strFormat, strFormat); String error = String.format("%s %s", format.print(now.minusDays(1)), format.print(now)); aJob.setCode(command); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals(false, actual.isSuccess()); assertEquals(true, actual.isFailed()); String expected = CallableScript.genErrorMessage(aJob, error+"\n"); assertEquals(expected, actual.getExceptionMessage().get()); }
@Test public void testJobResubmitSuccessUpdateQuery() throws BackendException { JobSpec aJob = TestAgent.getTestJob("Robert Frank", dao); String tableName = "table_dne_yet"; aJob.setCode(String.format("select * from %s", tableName)); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); Long nextId = cj.getJobId()+1; aJob = dao.getJob(aJob.getId()); aJob.setCode("show tables;"); dao.updateJob(aJob); TestAgent.runRunnable(consumer); doSleep(); assertEquals(0, dao.getRunningJobs().size()); assertEquals(2, consumer.getFinishedJobs(limit).size()); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); } @Test
@Test public void testGetFailedQueries() { JobSpec aJob = TestAgent.getTestJob("David Foster Wallace", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); assertEquals(true, cj.isDone()); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals(1, consumer.getFinishedJobs(limit).size()); }
"example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); Long id = cj.getJobId(); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals("successful Queries:" + success, 0, success.size()); assertEquals(false, consumer.getFinishedJobs(limit).get(nextId).isSuccess());
@Test public void testScriptRun() { JobSpec aJob = getTestScript("Chenoweth", dao); aJob.setCode("echo 'hi'; echo 'bye';"); dao.createJob(aJob); runRunnable(agentDriver); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals("", actual.getExceptionMessage().get()); assertEquals(true, actual.isSuccess()); }
public long createJobRun(CallableJob cq) { try { long jobId = createJobRun(new DateTime().withZone(DateTimeZone.UTC), cq); cq.setJobId(jobId); LOG.debug("Created jobRun: " + cq.getJobId()); return jobId; } catch (BackendException e) { throw new RuntimeException("Exception when updating jobRun: " + e.getMessage()); } }
synchronized public CallableJob getLatestMatching(PlannedJob pj, int limit) { Map<Long, CallableJob> toRet = new TreeMap<>(); Map<Long, CallableJob> runs = dao.getJobRuns(null, limit); for (Entry<Long, CallableJob> entry : runs.entrySet()) { Long key = entry.getKey(); CallableJob value = entry.getValue(); if (value.getPlannedJob().equals(pj)) { toRet.put(key, value); } } List<CallableJob> queries = new ArrayList<>(toRet.values()); if (queries.size() == 0) { return null; } Collections.reverse(queries); return queries.get(0); }
public void updateJobRun(CallableJob cq) { try { updateJobRun(new DateTime().withZone(DateTimeZone.UTC), cq); LOG.debug("Updated jobRun: " + cq.getJobId()); } catch (BackendException e) { throw new RuntimeException("Exception when updating jobRun: " + e.getMessage()); } }
private boolean areAllFuturesDone(AgentConsumer consumer){ boolean result = true; for (Entry<Long, CallableJob> i : dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).entrySet()){ if (!i.getValue().isDone()){ result = false; } } return result; }
@Test(timeout=2000) public void testScriptRunFail() { JobSpec aJob = getTestScript("Chenoweth", dao); String error = "error"; aJob.setCode(String.format("echo '%s' >&2; exit 1;", error)); dao.createJob(aJob); runRunnable(agentDriver); runRunnable(consumer); waitUntilJobsFinished(consumer, 1); CallableJob actual = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).get(1L); assertEquals(false, actual.isSuccess()); assertEquals(true, actual.isFailed()); String expected = CallableScript.genErrorMessage(aJob, error+"\n"); assertEquals(expected, actual.getExceptionMessage().get()); }
boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); Long nextId = cj.getJobId()+1; dao.execute( String.format("CREATE TABLE %s (blah TEXT)", tableName)); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); assertEquals(0, consumer.getFinishedJobs(limit).get(nextId).getStatus().get()); dao.execute( String.format("DROP TABLE IF EXISTS %s", tableName));
@Test public void testJobNoResubmit() throws BackendException { consumer.SLEEP_FOR = 1; JobSpec aJob = TestAgent.getTestJob("Hannah Arendt", dao); aJob.setCode("not a valid query..."); aJob.setShouldRerun(false); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); // let the job run and fail TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); // See if the job retries TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 1, jobRuns.values().size()); assertEquals(1, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); }
String name = value.getPlannedJob().getJobSpec().getName(); CallableJob cj = consumer.assembleCallableJob(value.getPlannedJob(), 1); consumer.submitJob(cj); Assert.assertEquals(name, rerun.getPlannedJob().getJobSpec().getName());
stat.setTimestamp(i++, new Timestamp(dt.getMillis())); stat.setString(i++, objToString(cj)); stat.setString(i++, cj.getPlannedJob().getJobSpec().getName()); stat.setString(i++, cj.getPlannedJob().getJobSpec().getCode()); stat.setInt(i++, cj.getStatus().get()); stat.setString(i++, cj.getExceptionMessage() != null ? cj.getExceptionMessage().get() : ""); stat.setTimestamp(i++, new Timestamp(cj.getStart().get())); stat.setTimestamp(i++, new Timestamp(cj.getFinish().get())); stat.setLong(i++, cj.getPlannedJob().getJobSpec().getId());
synchronized public Map<Long, CallableJob> getSuccesfulQueries(int limit) { Map<Long, CallableJob> toRet = new TreeMap<>(); Map<Long, CallableJob> runs = dao.getJobRuns(null, limit); for (Entry<Long, CallableJob> entry : runs.entrySet()) { Long key = entry.getKey(); CallableJob value = entry.getValue(); boolean isRunning = value.isRunning(); boolean isDone = value.isDone(); boolean isSuccess = value.isSuccess(); if (!isRunning && isDone && isSuccess) { toRet.put(key, value); } } return toRet; }