.setSqlQuery(query) .setQueryType(QueryType.UI_INTERNAL_PREVIEW) .setDatasetPath(NONE_PATH) .build(), new NoOpJobStatusListener() { @Override
.setSqlQuery(query) .setQueryType(queryType) .setDatasetPath(datasetPath.toNamespaceKey()) .setDatasetVersion(version) .build(), statusListener));
.setSqlQuery(new SqlQuery("select n_regionkey, max(n_nationkey) as max_nation from cp.\"tpch/nation.parquet\" group by n_regionkey", SYSTEM_USERNAME)) .setQueryType(QueryType.JDBC) .setDatasetPath(datsetPath.toNamespaceKey()) .setDatasetVersion(DatasetVersion.newVersion()) .build(), new JobStatusLogger() {
.setSqlQuery(getQueryFromConfig(dsGet)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(getDatasetPath(dsGet).toNamespaceKey()) .setDatasetVersion(v2) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary();
.setSqlQuery(getQueryFromConfig(ds1)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath1.toNamespaceKey()) .setDatasetVersion(ds1.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); .setSqlQuery(getQueryFromConfig(ds2)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath2.toNamespaceKey()) .setDatasetVersion(ds2.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); .setSqlQuery(getQueryFromConfig(ds3)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath3.toNamespaceKey()) .setDatasetVersion(ds3.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); .setSqlQuery(getQueryFromConfig(ds2)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath2.toNamespaceKey()) .setDatasetVersion(ds2.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary();
.setSqlQuery(getQueryFromConfig(dsGet)) .setQueryType(QueryType.UI_PREVIEW) .setDatasetPath(getDatasetPath(dsGet).toNamespaceKey()) .setDatasetVersion(v2) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); .setSqlQuery(new SqlQuery(dsGet.getSql(), dsGet.getContext(), USERNAME)) .setQueryType(QueryType.UI_PREVIEW) .setDatasetPath(getDatasetPath(dsGet).toNamespaceKey()) .setDatasetVersion(v2) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary();
.setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job2_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job3_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds3.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job1_3 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v2")).build(), NoOpJobStatusListener.INSTANCE); Job job2_2 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v2")).build(), NoOpJobStatusListener.INSTANCE); .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); job1_4.getData().loadIfNecessary();
.setSqlQuery(getQueryFromConfig(dsg1)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath.toNamespaceKey()) .setDatasetVersion(dsg1.getVersion()) .build(), NoOpJobStatusListener.INSTANCE);
.setSqlQuery(new SqlQuery(ds1.getSql(), ds1.getState().getContextList(), DEFAULT_USERNAME)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath.toNamespaceKey()) .setDatasetVersion(ds1.getVersion()) .build(), NoOpJobStatusListener.INSTANCE);
Job job1_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from LocalFS1.\"dac-sample1.json\" limit 10", ds1.toParentPathList(), DEFAULT_USERNAME)) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("1")) .build(), NoOpJobStatusListener.INSTANCE);
@Test public void testJobPhysicalDatasetParent() throws Exception { populateInitialData(); final DatasetPath datasetPath = new DatasetPath("DG.dsg1"); final DatasetUI dsg1 = getDataset(datasetPath); final JobsService jobsService = l(JobsService.class); Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromConfig(dsg1)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(datasetPath.toNamespaceKey()) .setDatasetVersion(dsg1.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); assertEquals(1, job.getJobAttempt().getInfo().getParentsList().size()); assertEquals(DEFAULT_USERNAME, job.getJobAttempt().getInfo().getUser()); assertEquals("/job/"+job.getJobId().getId()+"/data", JobResource.getPaginationURL(job.getJobId())); assertEquals(Arrays.asList("LocalFS1", "dac-sample1.json"), job.getJobAttempt().getInfo().getParentsList().get(0).getDatasetPathList()); assertEquals(DatasetType.PHYSICAL_DATASET_SOURCE_FILE, job.getJobAttempt().getInfo().getParentsList().get(0).getType()); @SuppressWarnings("unused") JobUI job1 = expectSuccess(getBuilder(getAPIv2().path("job/" + job.getJobId().getId())).buildGet(), JobUI.class); assertEquals(1, job.getJobAttempt().getInfo().getParentsList().size()); assertEquals(Arrays.asList("LocalFS1", "dac-sample1.json"), job.getJobAttempt().getInfo().getParentsList().get(0).getDatasetPathList()); assertEquals(DatasetType.PHYSICAL_DATASET_SOURCE_FILE, job.getJobAttempt().getInfo().getParentsList().get(0).getType()); JobDetailsUI jobDetails = expectSuccess(getBuilder(getAPIv2().path("job/" + job.getJobId().getId() + "/details")).buildGet(), JobDetailsUI.class); assertNotNull(jobDetails); assertEquals(JobResource.getPaginationURL(job.getJobId()), jobDetails.getPaginationUrl()); }
private RelNode getPlan(final String queryString) { final AtomicReference<RelNode> physical = new AtomicReference<>(null); final SqlQuery query = new SqlQuery(queryString, DEFAULT_USERNAME); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.ACCELERATOR_CREATE) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .setDatasetVersion(DatasetVersion.NONE) .build(), new NoOpJobStatusListener() { @Override public void planRelTransform(final PlannerPhase phase, final RelNode before, final RelNode after, final long millisTaken) { if (phase == PlannerPhase.PHYSICAL) { physical.set(after); } super.planRelTransform(phase, before, after, millisTaken); } }); job.getData().truncate(1); return physical.get(); } }
protected String getQueryPlan(final String query) { final AtomicReference<String> plan = new AtomicReference<>(""); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .build(), new NoOpJobStatusListener() { @Override public void planRelTransform(final PlannerPhase phase, final RelNode before, final RelNode after, final long millisTaken) { if (!Strings.isNullOrEmpty(plan.get())) { return; } if (phase == PlannerPhase.LOGICAL) { plan.set(RelOptUtil.dumpPlan("", after, SqlExplainFormat.TEXT, SqlExplainLevel.ALL_ATTRIBUTES)); } } }); job.getData().loadIfNecessary(); return plan.get(); }
public static Job submitRefreshJob(JobsService jobsService, NamespaceService namespaceService, ReflectionEntry entry, Materialization materialization, String sql, JobStatusListener jobStatusListener) { final SqlQuery query = new SqlQuery(sql, SYSTEM_USERNAME); NamespaceKey datasetPathList = new NamespaceKey(namespaceService.findDatasetByUUID(entry.getDatasetId()).getFullPathList()); MaterializationSummary materializationSummary = new MaterializationSummary() .setDatasetId(entry.getDatasetId()) .setReflectionId(entry.getId().getId()) .setLayoutVersion(entry.getTag()) .setMaterializationId(materialization.getId().getId()) .setReflectionName(entry.getName()) .setReflectionType(entry.getType().toString()); return jobsService.submitJob( JobRequest.newMaterializationJobBuilder(materializationSummary, new SubstitutionSettings(ImmutableList.of())) .setSqlQuery(query) .setQueryType(QueryType.ACCELERATOR_CREATE) .setDatasetPath(datasetPathList) .build(), jobStatusListener); }
private DatasetUI setupIteratorTests(String datasetName) throws Exception{ TestSpacesStoragePlugin.setup(getCurrentDremioDaemon()); DatasetUI dataset = getDataset(new DatasetPath(datasetName)); // run dataset twice. We do a run and a preview since subsequent previews won't actually rerun... getPreview(dataset); l(JobsService.class).submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromConfig(dataset)) .setQueryType(QueryType.UI_RUN) .setDatasetPath(getDatasetPath(dataset).toNamespaceKey()) .setDatasetVersion(dataset.getDatasetVersion()) .build(), NoOpJobStatusListener.INSTANCE) .getData() .loadIfNecessary(); return dataset; }
private static QueryProfile getQueryProfile(final String query) throws JobNotFoundException { final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .setDatasetVersion(DatasetVersion.NONE) .build(), new NoOpJobStatusListener()); job.getData().loadIfNecessary(); return getJobsService().getProfile(job.getJobId(), 0); }
@Test public void testMaterializationsTable() { jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("SELECT * FROM sys.materializations")) .setDatasetPath(NONE_PATH) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); } }
protected void setSystemOption(String optionName, String optionValue) { final String query = String.format("ALTER SYSTEM SET \"%s\"=%s", optionName, optionValue); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); }
@Test public void testReflectionsTable() { jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("SELECT * FROM sys.reflections")) .setDatasetPath(NONE_PATH) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); }