/** * @return Get query planning time. If the planning hasn't started, returns "Planning not started". If planning hasn't * completed, returns "Still planning". */ @SuppressWarnings("unused") public String getPlanningTime() { final QueryProfile profile = getProfile(); if (!profile.hasPlanningStart() || profile.getPlanningStart() == 0) { return "Planning not started"; } if (!profile.hasPlanningEnd() || profile.getPlanningEnd() == 0) { return "Still planning"; } // Starting from 3.0, the planning time includes the resource queueing time. Thus, correcting for it when resource scheduling time exists long planningPlusSchedulingTime = profile.getPlanningEnd() - profile.getPlanningStart(); UserBitShared.ResourceSchedulingProfile r = profile.getResourceSchedulingProfile(); if (r == null || r.getResourceSchedulingStart() == 0 || r.getResourceSchedulingEnd() == 0) { return NUMBER_FORMAT.format(planningPlusSchedulingTime) + "ms"; } long schedulingTime = r.getResourceSchedulingEnd() - r.getResourceSchedulingStart(); return NUMBER_FORMAT.format(planningPlusSchedulingTime - schedulingTime) + "ms"; }
public ProfileWrapper(final QueryProfile profile, boolean debug) { this.profile = profile; this.id = QueryIdHelper.getQueryId(profile.getId()); final List<MajorFragmentProfile> majors = new ArrayList<>(profile.getFragmentProfileList()); Collections.sort(majors, Comparators.majorId); fragmentProfiles.add(new FragmentWrapper(major, profile.getStart(), debug)); final List<NodeQueryProfile> nodeQueryProfiles = new ArrayList<>(profile.getNodeProfileList()); Collections.sort(nodeQueryProfiles, Comparators.endpoint); ows.add(new OperatorWrapper(ip.getLeft(), opmap.get(ip), profile.hasOperatorTypeMetricsMap() ? profile.getOperatorTypeMetricsMap(): null)); AccelerationDetails details = AccelerationDetailsUtils.deserialize(profile.getAccelerationProfile().getAccelerationDetails()); if (details != null && details.getReflectionRelationshipsList() != null) { wrapper = new AccelerationWrapper(details);
QueryProfile profile = recordProfile(zip, jobId, attemptIndex); if (profile.hasPrepareId()) { final QueryId id = profile.getPrepareId(); zip.putNextEntry(new ZipEntry(String.format("prepare_profile_attempt_%d.json", attemptIndex))); JobId prepareId = new JobId(new UUID(id.getPart1(), id.getPart2()).toString()); if (profile.getState() == UserBitShared.QueryResult.QueryState.FAILED) { zip.putNextEntry(new ZipEntry(String.format("log_attempt_%d.json", attemptIndex))); outIncludesLogs.value = recordLog(zip, userId, profile.getStart(), profile.getEnd(), jobId, submissionId);
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.QueryProfile other) { if (other == com.dremio.exec.proto.UserBitShared.QueryProfile.getDefaultInstance()) return this; if (other.hasId()) { mergeId(other.getId()); if (other.hasType()) { setType(other.getType()); if (other.hasStart()) { setStart(other.getStart()); if (other.hasEnd()) { setEnd(other.getEnd()); if (other.hasQuery()) { bitField0_ |= 0x00000010; query_ = other.query_; onChanged(); if (other.hasPlan()) { bitField0_ |= 0x00000020; plan_ = other.plan_; onChanged(); if (other.hasForeman()) { mergeForeman(other.getForeman()); if (other.hasState()) { setState(other.getState());
parsePhysicalPlan(); } catch (IOException ioe) { logger.error("Failed to parse physical plan for query {}, plan {}", jobId.getId(), queryProfile.getJsonPlan(), ioe); if (queryProfile.getFragmentProfileList() != null) { totalMajorFragments = queryProfile.getFragmentProfileList().size(); for (MajorFragmentProfile majorFragment: queryProfile.getFragmentProfileList()) { if (majorFragment.getMinorFragmentProfileList() == null) { continue;
if(message.hasId()) output.writeObject(1, message.getId(), com.dremio.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasType()) output.writeEnum(2, message.getType().getNumber(), false); if(message.hasStart()) output.writeInt64(3, message.getStart(), false); if(message.hasEnd()) output.writeInt64(4, message.getEnd(), false); if(message.hasQuery()) output.writeString(5, message.getQuery(), false); if(message.hasPlan()) output.writeString(6, message.getPlan(), false); if(message.hasForeman()) output.writeObject(7, message.getForeman(), com.dremio.exec.proto.SchemaCoordinationProtos.NodeEndpoint.WRITE, false); if(message.hasState()) output.writeEnum(8, message.getState().getNumber(), false); if(message.hasTotalFragments()) output.writeInt32(9, message.getTotalFragments(), false); if(message.hasFinishedFragments()) output.writeInt32(10, message.getFinishedFragments(), false); for(com.dremio.exec.proto.UserBitShared.MajorFragmentProfile fragmentProfile : message.getFragmentProfileList()) output.writeObject(11, fragmentProfile, com.dremio.exec.proto.SchemaUserBitShared.MajorFragmentProfile.WRITE, true); if(message.hasUser()) output.writeString(12, message.getUser(), false); if(message.hasError()) output.writeString(13, message.getError(), false);
@SuppressWarnings("unused") public String getResourceSchedulingOverview() { UserBitShared.ResourceSchedulingProfile r = profile.getResourceSchedulingProfile(); if (r == null) { return ""; if (profile.hasCancelReason()) { dlb.addItem("Cancellation Reason:", profile.getCancelReason());
final JobInfo jobInfo = jobAttempt.getInfo(); final QueryProfileParser profileParser = new QueryProfileParser(job.getJobId(), profile); jobInfo.setStartTime(profile.getStart()); jobInfo.setFinishTime(profile.getEnd()); if (profile.getResourceSchedulingProfile() != null) { if (jobInfo.getResourceSchedulingInfo() == null) { jobInfo.setResourceSchedulingInfo(new ResourceSchedulingInfo()); jobInfo.getResourceSchedulingInfo().setQueueName(profile.getResourceSchedulingProfile().getQueueName()); jobInfo.getResourceSchedulingInfo().setQueueId(profile.getResourceSchedulingProfile().getQueueId()); if (profile.hasError()) { jobInfo.setFailureInfo(profile.getError()); if (profile.hasVerboseError()) { jobInfo.setDetailedFailureInfo(JobsServiceUtil.toFailureInfo(profile.getVerboseError())); if (profile.hasCancelReason()) { final JobCancellationInfo cancellationInfo = new JobCancellationInfo(); cancellationInfo.setMessage(profile.getCancelReason()); jobInfo.setCancellationInfo(cancellationInfo);
@Override public void execStarted(QueryProfile profile) { try (TimedBlock b = Timer.time("execStarted")) { b.addID("attempt=" + attemptId); final JobInfo jobInfo = job.getJobAttempt().getInfo(); if(profile != null){ jobInfo.setStartTime(profile.getStart()); job.getJobAttempt().setState(JobState.RUNNING); final QueryProfileParser profileParser = new QueryProfileParser(jobId, profile); if (profile.getResourceSchedulingProfile() != null) { if (jobInfo.getResourceSchedulingInfo() == null) { jobInfo.setResourceSchedulingInfo(new ResourceSchedulingInfo()); } jobInfo.getResourceSchedulingInfo().setQueueName(profile.getResourceSchedulingProfile().getQueueName()); jobInfo.getResourceSchedulingInfo().setQueueId(profile.getResourceSchedulingProfile().getQueueId()); } job.getJobAttempt().setStats(profileParser.getJobStats()); job.getJobAttempt().setDetails(profileParser.getJobDetails()); storeJob(job); if (externalListenerManager != null) { externalListenerManager.queryUpdate(job); } } } catch (IOException e) { exception.addException(e); } }
@Test public void testResourceSchedulingInProfile() throws Exception { AttemptObservers observers = AttemptObservers.of(observer); final NodeEndpoint endpoint = NodeEndpoint.newBuilder().setAddress("host1").setFabricPort(12345).build(); when(context.getCurrentEndpoint()).thenReturn(endpoint); QueryManager queryManager = new QueryManager(queryId, context, null, completionListener, new Pointer<>(), observers, true, true, catalog); ResourceSchedulingDecisionInfo result = new ResourceSchedulingDecisionInfo(); result.setQueueId("abcd"); result.setQueueName("queue.abcd"); observers.resourcesScheduled(result); UserBitShared.QueryProfile queryProfile = queryManager.getQueryProfile("my description", UserBitShared.QueryResult.QueryState.RUNNING, null, "some reason"); assertNotNull(queryProfile.getResourceSchedulingProfile()); assertEquals("abcd", queryProfile.getResourceSchedulingProfile().getQueueId()); assertEquals("queue.abcd", queryProfile.getResourceSchedulingProfile().getQueueName()); } }
/** * <code>optional string cancel_reason = 35;</code> */ public Builder clearCancelReason() { bitField0_ = (bitField0_ & ~0x40000000); cancelReason_ = getDefaultInstance().getCancelReason(); onChanged(); return this; } /**
private void parsePlanningDetails() { if (queryProfile.getPlanningEnd() > 0 && queryProfile.getPlanningStart() > 0) { jobDetails.setTimeSpentInPlanning(queryProfile.getPlanningEnd() - queryProfile.getPlanningStart()); } else { jobDetails.setTimeSpentInPlanning(null); } // Not yet available. jobDetails.setPlansConsidered(null); }
private static MajorFragmentProfile findFragmentProfileWithId(QueryProfile profile, final int id) { return FluentIterable.from(profile.getFragmentProfileList()).firstMatch(new Predicate<MajorFragmentProfile>() { @Override public boolean apply(@Nullable MajorFragmentProfile input) { return input.getMajorFragmentId() == id; } }).get(); }
@Override public Object next() { QueryProfile profile = iter.next(); return new QueryInfo(profile.getForeman().getAddress(), profile.getUser(), QueryIdHelper.getQueryId(profile.getId()), profile.getQuery(), new Timestamp(profile.getStart())); }
@SuppressWarnings("unused") public String getQueueTime() { UserBitShared.ResourceSchedulingProfile r = profile.getResourceSchedulingProfile(); if (r == null || r.getResourceSchedulingStart() == 0 || r.getResourceSchedulingEnd() == 0) { return ""; } return NUMBER_FORMAT.format(r.getResourceSchedulingEnd() - r.getResourceSchedulingStart()) + "ms"; }
/** * <code>optional string error = 13;</code> */ public Builder clearError() { bitField0_ = (bitField0_ & ~0x00001000); error_ = getDefaultInstance().getError(); onChanged(); return this; } /**
public String getPerdiodFromStart(Long datetime) { if (datetime == null) { return ""; } return DurationFormatUtils.formatDurationWords( this.profile.getStart() - datetime, true, true); } }