final CompletionListener listener = new CompletionListener(); jobsService.get().submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INTERNAL_RUN)
return new JobRequest( requestType, sqlQuery,
final JobId jobId = JobsServiceUtil.getExternalIdAsJobId(externalId); final String inSpace = !jobRequest.getDatasetPathComponents().isEmpty() && namespaceService.exists(new NamespaceKey(jobRequest.getDatasetPathComponents().get(0)), NameSpaceContainer.Type.SPACE) ? jobRequest.getDatasetPathComponents().get(0) : null; final JobInfo jobInfo = jobRequest.asJobInfo(jobId, inSpace); final JobAttempt jobAttempt = new JobAttempt() .setInfo(jobInfo) final QueryType queryType = jobRequest.getQueryType(); final boolean enableLeafLimits = QueryTypeUtils.requiresLeafLimits(queryType); final LocalExecutionConfig config = .setUsername(jobRequest.getUsername()) .setSqlContext(jobRequest.getSqlQuery().getContext()) .setInternalSingleThreaded(queryType == UI_INITIAL_PREVIEW) .setQueryResultsStorePath(storageName) .setAllowPartitionPruning(queryType != QueryType.ACCELERATOR_EXPLAIN) .setExposeInternalSources(QueryTypeUtils.isInternal(queryType)) .setSubstitutionSettings(jobRequest.getSubstitutionSettings()) .build(); if (isPrepare) { queryRequest = CreatePreparedStatementReq.newBuilder() .setSqlQuery(jobRequest.getSqlQuery().getSql()) .build(); } else {
@Override public Job submitJob(JobRequest jobRequest, JobStatusListener statusListener) { checkNotNull(statusListener, "a status listener must be provided"); final Job job = startJob(jobRequest, statusListener); logger.debug("Submitted new job. Id: {} Type: {} Sql: {}", job.getJobId().getId(), jobRequest.getQueryType(), jobRequest.getSqlQuery()); return job; }
@POST @Path("file_preview/{path: .*}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public JobDataFragment previewFormatSettings(FileFormat fileFormat, @PathParam("path") String path) throws FileNotFoundException, SourceNotFoundException { FilePath filePath = FilePath.fromURLPath(homeName, path); logger.debug("filePath: " + filePath.toPathString()); // TODO, this should be moved to dataset resource and be paginated. SqlQuery query = new SqlQuery(format("select * from table(%s (%s)) limit 500", filePath.toPathString(), fileFormat.toTableOptions()), securityContext.getUserPrincipal().getName()); JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INITIAL_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE)); return job.getData().truncate(500); }
@POST @Path("file_preview_unsaved/{path: .*}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public JobDataFragment previewFormatSettingsStaging(FileFormat fileFormat, @PathParam("path") String path) throws FileNotFoundException, SourceNotFoundException { FilePath filePath = FilePath.fromURLPath(homeName, path); logger.debug("filePath: " + filePath.toPathString()); // use file's location directly to query file String fileLocation = PathUtils.toDottedPath(new org.apache.hadoop.fs.Path(fileFormat.getLocation())); SqlQuery query = new SqlQuery(format("select * from table(%s.%s (%s)) limit 500", SqlUtils.quoteIdentifier(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME), fileLocation, fileFormat.toTableOptions()), securityContext.getUserPrincipal().getName()); JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INITIAL_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE)); return job.getData().truncate(500); }
@POST @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public JobDataFragment query(CreateFromSQL sql) { SqlQuery query = new SqlQuery(sql.getSql(), sql.getContext(), securityContext); // Pagination is not supported in this API, so we need to truncate the results to 500 records return new JobUI(jobs.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.REST) .build(), NoOpJobStatusListener.INSTANCE)).getData().truncate(500); }
@Deprecated public JobDataFragment previewPhysicalDataset(String table, FileFormat formatOptions) { SqlQuery query = new SqlQuery(format("select * from table(%s (%s))", table, formatOptions.toTableOptions()), null, context.getUserPrincipal().getName()); // We still need to truncate the results to 500 as the preview physical datasets doesn't support pagination yet return new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INITIAL_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE)).getData().truncate(500); } }
/** * Get the preview response of dataset. Dataset could be a physical or virtual dataset. * @param limit Maximum number of records in initial response. * @return */ @GET @Path("preview") @Produces(APPLICATION_JSON) public InitialDataPreviewResponse preview(@QueryParam("limit") @DefaultValue("50") Integer limit) { final SqlQuery query = new SqlQuery( String.format("select * from %s", datasetPath.toPathString()), securityContext.getUserPrincipal().getName()); final JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE)); try { return InitialDataPreviewResponse.of(job.getData().truncate(limit)); } catch(UserException e) { throw DatasetTool.toInvalidQueryException(e, query.getSql(), ImmutableList.<String> of()); } }
@POST public QueryDetails runQuery(CreateFromSQL sql) { SqlQuery query = new SqlQuery(sql.getSql(), sql.getContext(), securityContext); Job job = jobs.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.REST) .build(), NoOpJobStatusListener.INSTANCE); return new QueryDetails(job.getJobId().getId()); } }
final Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INTERNAL_PREVIEW)
private JobData runExternalQuery(String sql) { return new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(sql, Arrays.asList("@" + DEFAULT_USER_NAME), DEFAULT_USER_NAME)) .build(), NoOpJobStatusListener.INSTANCE)).getData(); }
" port"; JobUI job = new JobUI(jobsService.get().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(sql, Arrays.asList("sys"), securityContext)) .setQueryType(QueryType.UI_INTERNAL_RUN)
@Test public void testGetJobResults() throws InterruptedException { JobsService jobs = l(JobsService.class); SqlQuery query = new SqlQuery("select * from sys.version", Collections.emptyList(), SystemUser.SYSTEM_USERNAME); Job job = jobs.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.REST) .build(), NoOpJobStatusListener.INSTANCE); String id = job.getJobId().getId(); while (true) { JobStatus status = expectSuccess(getBuilder(getPublicAPI(3).path(JOB_PATH).path(id)).buildGet(), JobStatus.class); JobState jobState = status.getJobState(); Assert.assertTrue("expected job to complete successfully", Arrays.asList(JobState.COMPLETED, JobState.RUNNING, JobState.ENQUEUED, JobState.STARTING).contains(jobState)); if (jobState == JobState.COMPLETED) { expectStatus(Response.Status.BAD_REQUEST, getBuilder(getPublicAPI(3).path(JOB_PATH).path(id).path("results").queryParam("limit", 1000)).buildGet()); break; } else { Thread.sleep(TimeUnit.MILLISECONDS.toMillis(100)); } } }
private JobDataFragment runQuery(String sql) { JobData completeJobData = new JobUI(jobs.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(sql, Collections.singletonList("cp"), DEFAULT_USERNAME)) .build(), NoOpJobStatusListener.INSTANCE)).getData(); return completeJobData.truncate(500); }
@Test public void testCancelJob() throws InterruptedException { JobsService jobs = l(JobsService.class); SqlQuery query = new SqlQuery("select * from sys.version", Collections.emptyList(), SystemUser.SYSTEM_USERNAME); Job job = jobs.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.REST) .build(), NoOpJobStatusListener.INSTANCE); String id = job.getJobId().getId(); expectSuccess(getBuilder(getPublicAPI(3).path(JOB_PATH).path(id).path("cancel")).buildPost(null)); while (true) { JobStatus status = expectSuccess(getBuilder(getPublicAPI(3).path(JOB_PATH).path(id)).buildGet(), JobStatus.class); JobState jobState = status.getJobState(); Assert.assertTrue("expected job to cancel successfully", Arrays.asList(JobState.RUNNING, JobState.ENQUEUED, JobState.STARTING, JobState.CANCELLATION_REQUESTED, JobState.CANCELED).contains(jobState)); if (jobState == JobState.CANCELED) { expectStatus(Response.Status.BAD_REQUEST, getBuilder(getPublicAPI(3).path(JOB_PATH).path(id).path("results").queryParam("limit", 1000)).buildGet()); assertEquals("Query cancelled by user 'dremio'", status.getCancellationReason()); break; } else if (jobState == JobState.COMPLETED) { // the job could complete before the cancel request, so make sure the test doesn't fail in that case. break; } else { Thread.sleep(TimeUnit.MILLISECONDS.toMillis(100)); } } }
protected void setSystemOption(String optionName, String optionValue) { final String query = String.format("ALTER SYSTEM SET \"%s\"=%s", optionName, optionValue); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); }
protected String getQueryPlan(final String query) { final AtomicReference<String> plan = new AtomicReference<>(""); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .build(), new NoOpJobStatusListener() { @Override public void planRelTransform(final PlannerPhase phase, final RelNode before, final RelNode after, final long millisTaken) { if (!Strings.isNullOrEmpty(plan.get())) { return; } if (phase == PlannerPhase.LOGICAL) { plan.set(RelOptUtil.dumpPlan("", after, SqlExplainFormat.TEXT, SqlExplainLevel.ALL_ATTRIBUTES)); } } }); job.getData().loadIfNecessary(); return plan.get(); }
@Test public void testExplain() throws Exception { SqlQuery ctas = getQueryFromSQL("EXPLAIN PLAN FOR SELECT * FROM sys.version"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testAlterOption() throws Exception { SqlQuery ctas = getQueryFromSQL("alter session set \"planner.enable_multiphase_agg\"=true"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }