@Override public Job getJob(JobId jobId, String userName) throws JobNotFoundException { return getJob(jobId); }
@Override public Iterable<Job> getJobsForDataset(final NamespaceKey datasetPath, int limit){ return getJobsForDataset(datasetPath, null, limit); }
@Override public Iterable<Job> getAllJobs(String filterString, String sortColumn, SortOrder sortOrder, int offset, int limit, String userName) { FindByCondition condition = new FindByCondition() .setOffset(offset) .setLimit(limit) .addSortings(buildSorter(sortColumn, sortOrder)) .setCondition(filterString, JobIndexKeys.MAPPING); return findJobs(condition); }
@Override public Iterable<Job> getJobsForDataset(final NamespaceKey datasetPath, final DatasetVersion version, String user, int limit){ FindByCondition condition = new FindByCondition() .setCondition(getFilter(datasetPath, version, user)) .setLimit(limit) .addSortings(DEFAULT_SORTER); return findJobs(condition); }
Job job1_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job2_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job3_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds3.toNamespaceKey()) assertEquals(1, jobsService.getJobsCount(ds1.toNamespaceKey())); assertEquals(1, jobsService.getJobsCount(ds2.toNamespaceKey())); assertEquals(1, jobsService.getJobsCount(ds3.toNamespaceKey())); Job job1_2 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v1")).build(), NoOpJobStatusListener.INSTANCE); Job job1_3 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds1.toNamespaceKey()) .setDatasetVersion(new DatasetVersion("v2")).build(), NoOpJobStatusListener.INSTANCE); Job job2_2 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("select * from LocalFS1.\"dac-sample1.json\" limit 1")) .setDatasetPath(ds2.toNamespaceKey()) assertEquals(3, jobsService.getJobsCount(ds1.toNamespaceKey()));
@Override public void recordExtraInfo(String name, byte[] bytes) { //TODO DX-10977 the reflection manager should rely on its own observer to store this information in a separate store if(job.getJobAttempt().getExtraInfoList() == null) { job.getJobAttempt().setExtraInfoList(new ArrayList<ExtraInfo>()); } job.getJobAttempt().getExtraInfoList().add(new ExtraInfo() .setData(ByteString.copyFrom(bytes)) .setName(name)); storeJob(job); super.recordExtraInfo(name, bytes); }
@Test public void testJobCleanup() throws Exception { jobsService = (LocalJobsService) l(JobsService.class); SqlQuery ctas = getQueryFromSQL("SHOW SCHEMAS"); Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); SabotContext context = l(SabotContext.class); OptionValue days = OptionValue.createLong(OptionType.SYSTEM, ExecConstants.RESULTS_MAX_AGE_IN_DAYS.getOptionName(), 0); context.getOptionManager().setOption(days); OptionValue millis = OptionValue.createLong(OptionType.SYSTEM, ExecConstants.DEBUG_RESULTS_MAX_AGE_IN_MILLISECONDS.getOptionName(), 10); context.getOptionManager().setOption(millis); Thread.sleep(20); LocalJobsService.CleanupTask cleanupTask = jobsService.new CleanupTask(); cleanupTask.cleanup(); //make sure that the job output directory is gone assertFalse(jobsService.getJobResultsStore().jobOutputDirectoryExists(job.getJobId())); job = jobsService.getJob(job.getJobId()); assertFalse(JobDetailsUI.of(job).getResultsAvailable()); context.getOptionManager().setOption(OptionValue.createLong(OptionType.SYSTEM, ExecConstants.RESULTS_MAX_AGE_IN_DAYS.getOptionName(), 30)); context.getOptionManager().setOption(OptionValue.createLong(OptionType.SYSTEM, ExecConstants.DEBUG_RESULTS_MAX_AGE_IN_MILLISECONDS.getOptionName(), 0)); }
@Override public QueryProfile getProfile(JobId jobId, int attempt) throws JobNotFoundException { Job job = getJob(jobId); final AttemptId attemptId = new AttemptId(JobsServiceUtil.getJobIdAsExternalId(jobId), attempt); if(jobIsDone(job.getJobAttempt())){ return profileStore.get(attemptId); } // Check if the profile for given attempt already exists. Even if the job is not done, it is possible that // profile exists for previous attempts final QueryProfile queryProfile = profileStore.get(attemptId); if (queryProfile != null) { return queryProfile; } final NodeEndpoint endpoint = job.getJobAttempt().getEndpoint(); if(endpoint.equals(identity)){ final ForemenTool tool = this.foremenTool.get(); Optional<QueryProfile> profile = tool.getProfile(attemptId.getExternalId()); return profile.orNull(); } try{ CoordTunnel tunnel = coordTunnelCreator.get().getTunnel(JobsServiceUtil.toPB(endpoint)); return tunnel.requestQueryProfile(attemptId.getExternalId()).checkedGet(15, TimeUnit.SECONDS); }catch(TimeoutException | RpcException | RuntimeException e){ logger.info("Unable to retrieve remote query profile for external id: {}", ExternalIdHelper.toString(attemptId.getExternalId()), e); return null; } }
@Test public void testJobParentSearch() throws Exception { Job jobA1 = createJob("A1", asList("space1", "ds1"), "v1", "A", "space1", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); jobA1.getJobAttempt().getInfo().setFieldOriginsList(asList( new FieldOrigin("foo") .setOriginsList(asList( new Origin("foo", false) .setTableList(asList("LocalFS1", "dac-sample1.json")) )) )); jobsService.storeJob(jobA1); List<Job> jobsForParent = ImmutableList.copyOf(jobsService.getJobsForParent(new NamespaceKey(asList("LocalFS1", "dac-sample1.json")), Integer.MAX_VALUE)); assertFalse(jobsForParent.isEmpty()); }
canceled += 1; jobsService.storeJob(jobA1); jobsService.storeJob(jobA2); jobsService.storeJob(jobA3); jobsService.storeJob(jobA4); jobsService.storeJob(jobA5); completed += 4; jobsService.storeJob(jobB1); jobsService.storeJob(jobB2); jobsService.storeJob(jobB3); jobsService.storeJob(jobB4); jobsService.storeJob(jobB5); jobsService.storeJob(jobC1); jobsService.storeJob(jobC2); jobsService.storeJob(jobC3); jobsService.storeJob(jobD1); completed += 1; jobs = ImmutableList.copyOf(jobsService.getAllJobs("st=ge=0", "st", SortOrder.ASCENDING, 0, Integer.MAX_VALUE, "A")); assertEquals(14, jobs.size());
@Override public List<Integer> getJobsCount(List<NamespaceKey> datasetPaths) { if (datasetPaths.isEmpty()) { return new ArrayList<>(0); } final List<SearchQuery> conditions = Lists.newArrayList(); for (NamespaceKey datasetPath: datasetPaths) { conditions.add(getFilter(datasetPath, null, null)); } return store.getCounts(conditions.toArray(new SearchQuery[conditions.size()])); }
@VisibleForTesting Iterable<Job> getAllJobs(){ FindByCondition condition = new FindByCondition(); condition.addSortings(DEFAULT_SORTER); return findJobs(condition); }
private static void deleteOldJobs(LocalKVStoreProvider provider, int maxDays) { System.out.print(String.format("Deleting jobs details & profiles older %d days... ", maxDays)); DeleteResult result = LocalJobsService.deleteOldJobs(provider, maxDays); System.out.println(String.format("Completed. Deleted %d jobs and %d profiles.", result.getJobsDeleted(), result.getProfilesDeleted())); }
private List<Job> getAllJobs(String filterString, String sortColumn, SortOrder sortOrder) { return ImmutableList.copyOf(jobsService.getAllJobs(filterString, sortColumn, sortOrder, 0, Integer.MAX_VALUE, DEFAULT_USERNAME)); }
new LocalJobsService( registry.getBindingCreator(), registry.provider(KVStoreProvider.class),
@Test // DX-5119 Index unquoted dataset names along with quoted ones. // TODO (Amit H): DX-1563 We should be using analyzer to match both rather than indexing twice. public void testUnquotedJobFilter() throws Exception { Job jobA1 = createJob("A1", Arrays.asList("Prod-Sample", "ds-1"), "v1", "A", "Prod-Sample", JobState.COMPLETED, "select * from LocalFS1.\"dac-sample1.json\"", 100L, 110L, QueryType.UI_RUN); jobsService.storeJob(jobA1); List<Job> jobs = getAllJobs("ads==Prod-Sample.ds-1", null, null); assertEquals(1, jobs.size()); jobs = getAllJobs("ds==Prod-Sample.ds-1", null, null); assertEquals(0, jobs.size()); }
@Override public Iterable<Job> getJobsForDataset(final NamespaceKey datasetPath, final DatasetVersion version, int limit){ FindByCondition condition = new FindByCondition() .setCondition(getFilter(datasetPath, version, null)) .setLimit(limit) .addSortings(DEFAULT_SORTER); return findJobs(condition); }
.put(any(JobId.class), any(JobResult.class)); LocalJobsService.setAbandonedJobsToFailedState(jobStore);
@Override public int getJobsCount(final NamespaceKey datasetPath) { return store.getCounts(getFilter(datasetPath, null, null)).get(0); }