@Before public void setup() throws Exception { clearAllDataExceptUser(); jobsService = (LocalJobsService) l(JobsService.class); }
@Test public void testCancel() throws Exception { final InitialPreviewResponse resp = createDatasetFromSQL(TestWebSocket.LONG_TEST_QUERY, null); final InitialTransformAndRunResponse runResp = expectSuccess( getBuilder( getAPIv2() .path(versionedResourcePath(resp.getDataset())) .path("transformAndRun") .queryParam("newVersion", newVersion()) ).buildPost(entity(new TransformSort("id", ASC), JSON)), InitialTransformAndRunResponse.class); JobId job = runResp.getJobId(); NotificationResponse response = expectSuccess( getBuilder( getAPIv2() .path("job") .path(job.getId()) .path("cancel") ).buildPost(entity(null, JSON)), NotificationResponse.class); assertEquals("Job cancellation requested", response.getMessage()); assertEquals(NotificationResponse.ResponseType.OK, response.getType()); }
@Test // DX-5119 Index unquoted dataset names along with quoted ones. // TODO (Amit H): DX-1563 We should be using analyzer to match both rather than indexing twice. public void testUnquotedJobFilter() throws Exception { Job jobA1 = createJob("A1", Arrays.asList("Prod-Sample", "ds-1"), "v1", "A", "Prod-Sample", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); jobsService.storeJob(jobA1); List<Job> jobs = getAllJobs("ads==Prod-Sample.ds-1", null, null); assertEquals(1, jobs.size()); jobs = getAllJobs("ds==Prod-Sample.ds-1", null, null); assertEquals(0, jobs.size()); }
@Test public void testCTASReplace() throws Exception { NamespaceKey namespaceKey = new NamespaceKey("ctasSpace"); SpaceConfig spaceConfig = new SpaceConfig(); spaceConfig.setName("ctasSpace"); newNamespaceService().addOrUpdateSpace(namespaceKey, spaceConfig); SqlQuery ctas = getQueryFromSQL("CREATE OR REPLACE VIEW ctasSpace.ctastest AS select * from (VALUES (1))"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); waitForCompletion(ctasJob); ctas = getQueryFromSQL("CREATE OR REPLACE VIEW ctasSpace.ctastest AS select * from (VALUES (2))"); ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); waitForCompletion(ctasJob); newNamespaceService().deleteSpace(namespaceKey, newNamespaceService().getSpace(namespaceKey).getTag()); }
@Ignore @Test public void testJobService() throws Exception { populateInitialData(); .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job2_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job3_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds3.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job1_3 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v2")).build(), NoOpJobStatusListener.INSTANCE); Job job2_2 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v2")).build(), NoOpJobStatusListener.INSTANCE); .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1"))
final Job job = createJob("A1", Arrays.asList("space1", "ds1"), "v1", "A", "space1", JobState.FAILED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); AttemptId attemptId = AttemptId.of(externalId); job.getJobAttempt() attemptId = attemptId.nextAttempt(); final JobAttempt jobAttempt = new JobAttempt() .setInfo(newJobInfo(job.getJobAttempt().getInfo(), 100L+2*i, 100L+2*i+1, "failed")) .setAttemptId(AttemptIdUtils.toString(attemptId)) .setState(JobState.FAILED) .setInfo(newJobInfo(job.getJobAttempt().getInfo(), 106, 107, null)) .setAttemptId(AttemptIdUtils.toString(attemptId)) .setState(JobState.COMPLETED) final AttemptDetailsUI attemptDetails = detailsUI.getAttemptDetails().get(i); final String reason = i == 0 ? "" : (i == 1 ? AttemptsUIHelper.OUT_OF_MEMORY_TEXT : AttemptsUIHelper.SCHEMA_CHANGE_TEXT); checkAttemptDetail(attemptDetails, job.getJobId(), i, i == 4 ? JobState.COMPLETED : JobState.FAILED, reason);
@Test public void testCTASAndDropTable() throws Exception { // Create a table SqlQuery ctas = getQueryFromSQL("CREATE TABLE \"$scratch\".\"ctas\" AS select * from cp.\"json/users.json\" LIMIT 1"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); FileSystemPlugin plugin = (FileSystemPlugin) getCurrentDremioDaemon().getBindingProvider().lookup(CatalogService.class).getSource("$scratch"); // Make sure the table data files exist File ctasTableDir = new File(plugin.getConfig().getPath().toString(), "ctas"); assertTrue(ctasTableDir.exists()); assertTrue(ctasTableDir.list().length >= 1); // Now drop the table SqlQuery dropTable = getQueryFromSQL("DROP TABLE \"$scratch\".\"ctas\""); Job dropTableJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(dropTable) .setQueryType(QueryType.ACCELERATOR_DROP) .build(), NoOpJobStatusListener.INSTANCE); dropTableJob.getData().loadIfNecessary(); // Make sure the table data directory is deleted assertFalse(ctasTableDir.exists()); }
@Test public void testJobCleanup() throws Exception { jobsService = (LocalJobsService) l(JobsService.class); SqlQuery ctas = getQueryFromSQL("SHOW SCHEMAS"); Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); SabotContext context = l(SabotContext.class); OptionValue days = OptionValue.createLong(OptionType.SYSTEM, ExecConstants.RESULTS_MAX_AGE_IN_DAYS.getOptionName(), 0); context.getOptionManager().setOption(days); OptionValue millis = OptionValue.createLong(OptionType.SYSTEM, ExecConstants.DEBUG_RESULTS_MAX_AGE_IN_MILLISECONDS.getOptionName(), 10); context.getOptionManager().setOption(millis); Thread.sleep(20); LocalJobsService.CleanupTask cleanupTask = jobsService.new CleanupTask(); cleanupTask.cleanup(); //make sure that the job output directory is gone assertFalse(jobsService.getJobResultsStore().jobOutputDirectoryExists(job.getJobId())); job = jobsService.getJob(job.getJobId()); assertFalse(JobDetailsUI.of(job).getResultsAvailable()); context.getOptionManager().setOption(OptionValue.createLong(OptionType.SYSTEM, ExecConstants.RESULTS_MAX_AGE_IN_DAYS.getOptionName(), 30)); context.getOptionManager().setOption(OptionValue.createLong(OptionType.SYSTEM, ExecConstants.DEBUG_RESULTS_MAX_AGE_IN_MILLISECONDS.getOptionName(), 0)); }
@Test public void testAlterOption() throws Exception { SqlQuery ctas = getQueryFromSQL("alter session set \"planner.enable_multiphase_agg\"=true"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testJobParentSearch() throws Exception { Job jobA1 = createJob("A1", asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); jobA1.getJobAttempt().getInfo().setFieldOriginsList(asList( new FieldOrigin("foo") .setOriginsList(asList( new Origin("foo", false) .setTableList(asList("LocalFS1", "dac-sample1.json")) )) )); jobsService.storeJob(jobA1); List<Job> jobsForParent = ImmutableList.copyOf(jobsService.getJobsForParent(new NamespaceKey(asList("LocalFS1", "dac-sample1.json")), Integer.MAX_VALUE)); assertFalse(jobsForParent.isEmpty()); }
@Test public void testExplain() throws Exception { SqlQuery ctas = getQueryFromSQL("EXPLAIN PLAN FOR SELECT * FROM sys.version"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testSingleCompletedAttempt() throws Exception { final String attemptId = AttemptIdUtils.toString(new AttemptId()); Job job = createJob("A1", Arrays.asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); job.getJobAttempt().setDetails(new JobDetails()); job.getJobAttempt().setAttemptId(attemptId); JobDetailsUI detailsUI = new JobDetailsUI(job.getJobId(), job.getJobAttempt().getDetails(), JobResource.getPaginationURL(job.getJobId()), job.getAttempts(), JobResource.getDownloadURL(job), null, null, null, true, null, null); assertEquals("", detailsUI.getAttemptsSummary()); assertEquals(1, detailsUI.getAttemptDetails().size()); AttemptDetailsUI attemptDetailsUI = detailsUI.getAttemptDetails().get(0); assertEquals("", attemptDetailsUI.getReason()); assertEquals(JobState.COMPLETED, attemptDetailsUI.getResult()); assertEquals("/profiles/" + job.getJobId().getId() + "?attempt=0", attemptDetailsUI.getProfileUrl()); }
Job jobA1 = createJob("A1", Arrays.asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); Job jobA2 = createJob("A2", Arrays.asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 250L, 260L, QueryType.UI_RUN); Job jobA3 = createJob("A3", Arrays.asList("space1", "ds1"), "v2", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 300L, 400L, QueryType.UI_RUN); Job jobA4 = createJob("A4", Arrays.asList("space1", "ds1"), "v3", "A", "space1", JobState.RUNNING, "select * from LocalFS1.\"dac-sample1.json\"", 100L, null, QueryType.UI_PREVIEW); Job jobA5 = createJob("A5", Arrays.asList("space1", "ds1"), "v2", "A", "space1", JobState.CANCELED, "select * from LocalFS1.\"dac-sample1.json\"", 300L, 301L, QueryType.UI_INTERNAL_PREVIEW); Job jobB1 = createJob("B1", Arrays.asList("space1", "ds2"), "v1", "B", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 120L, QueryType.UI_PREVIEW); Job jobB2 = createJob("B2", Arrays.asList("space1", "ds2"), "v2", "B", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 230L, 290L, QueryType.UI_PREVIEW); Job jobB3 = createJob("B3", Arrays.asList("space1", "ds2"), "v2", "B", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 300L, 400L, QueryType.UNKNOWN); Job jobB4 = createJob("B4", Arrays.asList("space1", "ds2"), "v2", "B", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 1000L, 2000L, QueryType.UNKNOWN); Job jobB5 = createJob("B5", Arrays.asList("space1", "ds2"), "v3", "B", "space1", JobState.RUNNING, "select * from LocalFS1.\"dac-sample1.json\"", 300L, null, QueryType.UI_INTERNAL_PREVIEW); Job jobC1 = createJob("C1", Arrays.asList("space2", "ds3"), "v1", "C", "space2", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 400L, 500L, QueryType.UI_RUN); Job jobC2 = createJob("C2", Arrays.asList("space2", "ds3"), "v1", "C", "space2", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 500L, 600L, QueryType.UI_RUN); Job jobC3 = createJob("C3", Arrays.asList("space2", "ds3"), "v2", "C", "space2", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 600L, 700L, QueryType.UI_PREVIEW); Job jobD1 = createJob("D1", Arrays.asList("space3","ds4"), "v4", "D", "space3", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 10L, 7000L, QueryType.REST); jobsService.storeJob(jobD1); completed += 1; List<Job> jobs = getAllJobs("spc==space1", null, null); assertEquals(10, jobs.size()); jobs = getAllJobs("spc==space2", null, null); assertEquals(3, jobs.size()); jobs = getAllJobs("spc==space3", null, null); assertEquals(1, jobs.size()); jobs = getAllJobs("qt==UI_RUN", null, null); assertEquals(5, jobs.size());
@Test public void testAliasedQuery() throws Exception { SqlQuery ctas = getQueryFromSQL("SHOW SCHEMAS"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testSingleFailedAttempt() throws Exception { final String attemptId = AttemptIdUtils.toString(new AttemptId()); Job job = createJob("A1", Arrays.asList("space1", "ds1"), "v1", "A", "space1", JobState.FAILED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); job.getJobAttempt().setDetails(new JobDetails()); job.getJobAttempt().setAttemptId(attemptId); JobDetailsUI detailsUI = new JobDetailsUI(job.getJobId(), job.getJobAttempt().getDetails(), JobResource.getPaginationURL(job.getJobId()), job.getAttempts(), JobResource.getDownloadURL(job), new JobFailureInfo("Some error message", JobFailureType.UNKNOWN, null), null, null, false, null, null); assertEquals("", detailsUI.getAttemptsSummary()); assertEquals(1, detailsUI.getAttemptDetails().size()); AttemptDetailsUI attemptDetailsUI = detailsUI.getAttemptDetails().get(0); assertEquals("", attemptDetailsUI.getReason()); assertEquals(JobState.FAILED, attemptDetailsUI.getResult()); assertEquals("/profiles/" + job.getJobId().getId() + "?attempt=0", attemptDetailsUI.getProfileUrl()); }