public CallableQuery(PlannedJob plannedJob, JobDao dao, Reporting reporting, String hostname, MailInfo mailInfo, Session session, SupportedDriver driver, String reportRootPath, int attemptNumber) { this.plannedJob = plannedJob; this.dao = dao; this.reporting = reporting; this.hostname = hostname; this.mailInfo = mailInfo; this.session = session; this.driver = driver; this.reportRootPath = reportRootPath; this.attemptNumber = attemptNumber; setReplacedCode(); if (plannedJob.getJobSpec().getResultQuery() != null) { this.replacedReportQuery = QueryReplaceUtil.replaceDateValues( plannedJob.getJobSpec().getResultQuery(), plannedJob.getReplaceTime()); } else { this.replacedReportQuery = null; } this.parts = cleanupQuery(replacedCode.split(QUERY_SPLITTER)); }
@VisibleForTesting public static String createMessageContent(PersistentResultSet results, JobSpec spec, String replacedReportQuery) { StringBuilder sb = new StringBuilder(); sb.append("<br>"); sb.append(createInlineResults(results, spec)); sb.append("<br>"); sb.append("<pre> Query: " + replacedReportQuery + "</pre>"); sb.append("<br>"); sb.append("Note: For brevity the first " + MAX_RESULTS_IN_BODY + " rows are included in the email body."); return sb.toString(); }
@Override protected void callInternal() throws BackendException { JobSpec currJob = plannedJob.getJobSpec(); String jobName = currJob.getName(); try (Connection conn = getConnectionForJobSpec(currJob)) { int step = 0; try (Statement statement = conn.createStatement()) { while (step < parts.length) { doStep(currJob, parts[step], statement); step++; } } catch (SQLException ex) { throw new BackendException(ex); } if (replacedReportQuery != null && !replacedReportQuery.isEmpty()) { PersistentResultSet results = doReportStep(conn, replacedReportQuery); String content = createMessageContent (results, currJob, this.getReplacedReportQuery()); DataSource attachment = createAttachment(results); if (reportRootPath != null) { writeReportToLocal(results, reportRootPath, plannedJob); } sendEmail(mailInfo, attachment, content, currJob); } } catch (Exception ex) { throw new RuntimeException(ex); } reporting.mark("chronos.query." + jobName + "." + "passed"); setStatus(0); }
@VisibleForTesting public static void writeReportToLocal(PersistentResultSet result, String rootPath, PlannedJob plannedJob) { String reportJobPath = getJobReportDir(rootPath, plannedJob); String reportPath = getJobReportPath(reportJobPath, plannedJob);
PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), folder.getRoot().getPath(), 1); CallableQuery.getJobReportDir(consumer.getReportRootPath(), pj); String reportPath = CallableQuery.getJobReportPath(reportJobPath, pj); assertEquals(true, cj.isDone()); assertEquals(true, cj.isSuccess()); assertNotNull(folder.getRoot().list()); String actualPath =
dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); results.getData().get(0).get(0), results.getData().get(0).get(1), results.getData().get(1).get(0), results.getData().get(1).get(1), cq.getReplacedReportQuery()); String actual = CallableQuery.createMessageContent(results, aJob, cq.getReplacedReportQuery()); Assert.assertEquals(expected, actual);
public CallableJob assembleCallableJob(PlannedJob plannedJob, int attemptNumber) { switch(plannedJob.getJobSpec().getType()) { case Query: { SupportedDriver driver = SupportedDriver.getSupportedDriverFromString( plannedJob.getJobSpec().getDriver(), drivers); return new CallableQuery(plannedJob, dao, reporter, hostname, mailInfo, session, driver, reportRootPath, attemptNumber); } case Script: return new CallableScript(plannedJob, dao, reporter, -1L, hostname, mailInfo, session, attemptNumber); default: throw new UnsupportedOperationException( String.format("Unknown jobtype...%s", plannedJob.getJobSpec().getType())); } }
CallableQuery callableQuery = new CallableQuery(); PersistentResultSet result = new PersistentResultSet(); result.setColumnNames(Arrays.asList("entry_id", "title", "social")); callableQuery.writeReportToLocal(result, rootPath, plannedJob);
@Test public void testReplacedReportQuery() { JobSpec aJob = TestAgent.getTestJob("Mark Rothko", dao); aJob.setResultQuery("select * from blah where dt=${YYYMMDDHH}"); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableQuery cq = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); String expected = "select * from blah where dt=1970010100"; Assert.assertEquals(expected, cq.getReplacedReportQuery()); }
@CoverageIgnore private static DataSource createAttachment(PersistentResultSet results) { try { String text = makeAttachmentText(results); DataSource source = new ByteArrayDataSource(text, TSV); return source; } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testCleanupQuery() { for (String s : new String[]{ "\r", "\n"}) { String q1 = "scat boo dee dat"; String q2 = "badoo dat dat doo"; String aQueryWithExtraSpace = String.format("%s%s%s;%s%s%s; ", s, q1, s, s, q2, s); String[] actual = CallableQuery.cleanupQuery( aQueryWithExtraSpace.split(CallableQuery.QUERY_SPLITTER)); String[] expected = new String[]{ " " + q1 + " ", " " + q2 + " " }; assertArrayEquals(expected, actual); } }
@Override public Void call() throws Exception { try { super.call(); } catch (Exception ex) { ex.printStackTrace(); } finally { try { Thread.sleep(sleepFor); } catch (InterruptedException ex) { CallableQuery.LOG.debug("interrupted"); } catch (Exception ex) { ex.printStackTrace(); } finish.set(System.currentTimeMillis()); sleepyFinish.set(System.currentTimeMillis()); dao.updateJobRun(this); } return null; }
@Test public void testGetJobRuns() { List<Long> ids = new ArrayList<>(); Map<Long, CallableJob> expected = new HashMap<>(); for (String name : new String[] { "Wifredo Lam", "Rene Magritte" }) { JobSpec aJob = TestAgent.getTestJob(name, dao); try { dao.createJob(aJob); aJob = dao.getJob(aJob.getId()); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, null, "example.com", null, null, null, null, 1); dao.createJobRun(cj); ids.add(aJob.getId()); expected.put(aJob.getId(), cj); } catch (Exception ex) { ex.printStackTrace(); } } for (Long id : ids) { Map<Long, CallableJob> got = dao.getJobRuns(id, limit); assertEquals(1, got.size()); assertEquals(expected.get(id), got.get(id)); } Map<Long, CallableJob> got = dao.getJobRuns(null, limit); assertEquals(expected, got); }
@Test public void testMakeAttachmentText() { PersistentResultSet results = getPRS(); String actual = CallableQuery.makeAttachmentText(results); String expected = String.format("%s(%s) %s(%s)\n" + "%s %s\n" + "%s %s\n", results.getColumnNames().get(0), results.getColumnTypes().get(0), results.getColumnNames().get(1), results.getColumnTypes().get(1), results.getData().get(0).get(0), results.getData().get(0).get(1), results.getData().get(1).get(0), results.getData().get(1).get(1)); Assert.assertEquals(expected, actual); } }
@Test public void testPersistJobRuns() { JobSpec aJob = TestAgent.getTestJob("Roxane Gay", dao); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitUntilJobsFinished(consumer, 1); assertEquals(1, consumer.getSuccesfulQueries(limit).size()); // Let job run and result doesn't matter // Now we should be able to get job history from a new consumer // and the job above should exist! JobDao local = new H2TestJobDaoImpl(); local.setDataSource(H2TestUtil.getDataSource()); assertEquals(1, local.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS).values().size()); }
@Test public void testResultEmail(){ PersistentResultSet rs = new PersistentResultSet(); rs.setColumnNames(Arrays.asList("a", "b")); rs.setColumnTypes(Arrays.asList("string", "string")); List<List<Object>> a = new ArrayList<>(); List<Object> k = new ArrayList<>(); k.add("a");k.add("b"); List<Object> l = new ArrayList<>(); l.add("c");l.add("d"); a.add(k); a.add(l); rs.setData(a); JobSpec js = new JobSpec(); StringBuilder s = CallableQuery.createInlineResults(rs, js); Assert.assertFalse(s.toString().equals("")); }
PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj);
@Test public void testGetFailedQueries() { JobSpec aJob = TestAgent.getTestJob("David Foster Wallace", dao); aJob.setCode("not a valid query..."); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); TestAgent.waitForFail(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); assertEquals(true, cj.isDone()); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); assertEquals(1, consumer.getFinishedJobs(limit).size()); }
@Test public void testJobNoResubmit() throws BackendException { consumer.SLEEP_FOR = 1; JobSpec aJob = TestAgent.getTestJob("Hannah Arendt", dao); aJob.setCode("not a valid query..."); aJob.setShouldRerun(false); dao.createJob(aJob); PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj); // let the job run and fail TestAgent.waitUntilJobsFinished(consumer, 1); boolean isSuccess = cj.isSuccess(); assertEquals(false, isSuccess); Map<Long, CallableJob> expected = new HashMap<>(); expected.put(cj.getJobId(), cj); assertEquals(expected, consumer.getFailedQueries(limit)); // See if the job retries TestAgent.runRunnable(consumer); TestAgentConsumer.doSleep(); assertEquals(0, dao.getRunningJobs().size()); Map<Long, CallableJob> jobRuns = dao.getJobRuns(null, AgentConsumer.LIMIT_JOB_RUNS); assertEquals("jobRuns: " + jobRuns, 1, jobRuns.values().size()); assertEquals(1, consumer.getFinishedJobs(limit).size()); assertEquals(0, consumer.getSuccesfulQueries(limit).size()); }
PlannedJob pj = new PlannedJob(aJob, Utils.getCurrentTime()); CallableJob cj = new CallableQuery(pj, dao, reporting, "example.com", mailInfo, null, drivers.get(0), null, 1); consumer.submitJob(cj);