public void queryCreatedEvent(BasicQueryInfo queryInfo) { eventListenerManager.queryCreated( new QueryCreatedEvent( queryInfo.getQueryStats().getCreateTime().toDate().toInstant(), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), new QueryMetadata( queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), QUEUED.toString(), queryInfo.getSelf(), Optional.empty(), Optional.empty()))); }
private static QueryStateInfo createQueryStateInfo( BasicQueryInfo queryInfo, Optional<ResourceGroupId> groupId, Optional<List<ResourceGroupInfo>> pathToRoot, Optional<QueryProgressStats> progress) { return new QueryStateInfo( queryInfo.getQueryId(), queryInfo.getState(), groupId, queryInfo.getQuery(), queryInfo.getQueryStats().getCreateTime(), queryInfo.getSession().getUser(), queryInfo.getSession().getSource(), queryInfo.getSession().getClientInfo(), queryInfo.getSession().getCatalog(), queryInfo.getSession().getSchema(), pathToRoot, progress); }
@Override public BasicQueryInfo getBasicQueryInfo() { return new BasicQueryInfo(getQueryInfo()); }
public static QueryStateInfo createQueryStateInfo(BasicQueryInfo queryInfo, Optional<ResourceGroupId> group) { Optional<QueryProgressStats> progress = Optional.empty(); if (!queryInfo.getState().isDone() && queryInfo.getState() != QUEUED) { progress = Optional.of(createQueryProgressStats(queryInfo.getQueryStats())); } return createQueryStateInfo(queryInfo, group, Optional.empty(), progress); }
private QueryStateInfo getQueryStateInfo(BasicQueryInfo queryInfo) { Optional<ResourceGroupId> groupId = queryInfo.getResourceGroupId(); if (queryInfo.getState() == QUEUED) { return createQueuedQueryStateInfo( queryInfo, groupId, groupId.map(resourceGroupManager::getPathToRoot)); } return createQueryStateInfo(queryInfo, groupId); }
BasicQueryInfo basicInfo = new BasicQueryInfo( new QueryInfo( new QueryId("0"), assertEquals(basicInfo.getQueryId().getId(), "0"); assertEquals(basicInfo.getState(), RUNNING); assertEquals(basicInfo.getMemoryPool().getId(), "reserved"); assertEquals(basicInfo.isScheduled(), false); assertEquals(basicInfo.getQuery(), "SELECT 4"); assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30")); assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30")); assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m")); assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m")); assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16); assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17); assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18); assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19); assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0); assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB")); assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("23GB")); assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m")); assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true); assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY)); assertEquals(basicInfo.getQueryStats().getProgressPercentage(), OptionalDouble.of(100));
private static void logQueryTimeline(BasicQueryInfo queryInfo) { DateTime queryStartTime = queryInfo.getQueryStats().getCreateTime(); DateTime queryEndTime = queryInfo.getQueryStats().getEndTime(); // query didn't finish cleanly if (queryStartTime == null || queryEndTime == null) { return; } long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0); logQueryTimeline( queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, elapsed, 0, 0, 0, queryStartTime, queryEndTime); }
private void cancelAllQueries() { QueryManager queryManager = coordinator.getQueryManager(); for (BasicQueryInfo queryInfo : queryManager.getQueries()) { if (!queryInfo.getState().isDone()) { queryManager.cancelQuery(queryInfo.getQueryId()); } } }
@GET public List<BasicQueryInfo> getAllQueryInfo(@QueryParam("state") String stateFilter) { QueryState expectedState = stateFilter == null ? null : QueryState.valueOf(stateFilter.toUpperCase(Locale.ENGLISH)); ImmutableList.Builder<BasicQueryInfo> builder = new ImmutableList.Builder<>(); for (BasicQueryInfo queryInfo : queryManager.getQueries()) { if (stateFilter == null || queryInfo.getState() == expectedState) { builder.add(queryInfo); } } return builder.build(); }
private void waitForQueryToBeKilled(DistributedQueryRunner queryRunner) throws InterruptedException { while (true) { for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) { if (info.getState().isDone()) { assertNotNull(info.getErrorCode()); assertEquals(info.getErrorCode().getCode(), CLUSTER_OUT_OF_MEMORY.toErrorCode().getCode()); return; } } MILLISECONDS.sleep(10); } }
@GET @Produces(MediaType.APPLICATION_JSON) public List<QueryStateInfo> getQueryStateInfos(@QueryParam("user") String user) { List<BasicQueryInfo> queryInfos = queryManager.getQueries(); if (!isNullOrEmpty(user)) { queryInfos = queryInfos.stream() .filter(queryInfo -> Pattern.matches(user, queryInfo.getSession().getUser())) .collect(toImmutableList()); } return queryInfos.stream() .filter(queryInfo -> !queryInfo.getState().isDone()) .map(this::getQueryStateInfo) .collect(toImmutableList()); }
assertFalse(info.getState().isDone()); assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool()); currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries(); for (BasicQueryInfo info : currentQueryInfos) { assertFalse(info.getState().isDone()); assertEquals(info.getState(), FINISHED);
private Set<String> captureQueries(Callable<?> action) throws Exception { Set<QueryId> queryIdsBefore = server.getQueryManager().getQueries().stream() .map(BasicQueryInfo::getQueryId) .collect(toImmutableSet()); action.call(); return server.getQueryManager().getQueries().stream() .filter(queryInfo -> !queryIdsBefore.contains(queryInfo.getQueryId())) .map(BasicQueryInfo::getQuery) .collect(toImmutableSet()); }
private Response failQuery(QueryId queryId, PrestoException queryException) { requireNonNull(queryId, "queryId is null"); try { QueryState state = queryManager.getQueryState(queryId); // check before killing to provide the proper error code (this is racy) if (state.isDone()) { return Response.status(Status.CONFLICT).build(); } queryManager.failQuery(queryId, queryException); // verify if the query was failed (if not, we lost the race) if (!queryException.getErrorCode().equals(queryManager.getQueryInfo(queryId).getErrorCode())) { return Response.status(Status.CONFLICT).build(); } return Response.status(Status.OK).build(); } catch (NoSuchElementException e) { return Response.status(Status.GONE).build(); } }
private static boolean isBlockedWaitingForMemory(BasicQueryInfo info) { BasicQueryStats stats = info.getQueryStats(); boolean isWaitingForMemory = stats.getBlockedReasons().contains(WAITING_FOR_MEMORY); if (!isWaitingForMemory) { return false; } // queries are not marked as fully blocked if there are no running drivers return stats.isFullyBlocked() || stats.getRunningDrivers() == 0; }
assertEquals(queryInfo.getSession().getSystemProperties(), ImmutableMap.builder() .put(QUERY_MAX_MEMORY, "1GB") .put(JOIN_DISTRIBUTION_TYPE, "partitioned") assertEquals(queryInfo.getSession().getClientInfo().get(), "{\"clientVersion\":\"testVersion\"}"); assertEquals(queryInfo.getSession().getPreparedStatements(), ImmutableMap.builder() .put("foo", "select * from bar") .build());
BasicQueryInfo basicInfo = new BasicQueryInfo( new QueryInfo( new QueryId("0"), assertEquals(basicInfo.getQueryId().getId(), "0"); assertEquals(basicInfo.getState(), RUNNING); assertEquals(basicInfo.getMemoryPool().getId(), "reserved"); assertEquals(basicInfo.isScheduled(), false); assertEquals(basicInfo.getQuery(), "SELECT 4"); assertEquals(basicInfo.getQueryStats().getCreateTime(), DateTime.parse("1991-09-06T05:00-05:30")); assertEquals(basicInfo.getQueryStats().getEndTime(), DateTime.parse("1991-09-06T06:00-05:30")); assertEquals(basicInfo.getQueryStats().getElapsedTime(), Duration.valueOf("8m")); assertEquals(basicInfo.getQueryStats().getExecutionTime(), Duration.valueOf("44m")); assertEquals(basicInfo.getQueryStats().getTotalDrivers(), 16); assertEquals(basicInfo.getQueryStats().getQueuedDrivers(), 17); assertEquals(basicInfo.getQueryStats().getRunningDrivers(), 18); assertEquals(basicInfo.getQueryStats().getCompletedDrivers(), 19); assertEquals(basicInfo.getQueryStats().getCumulativeUserMemory(), 20.0); assertEquals(basicInfo.getQueryStats().getUserMemoryReservation(), DataSize.valueOf("21GB")); assertEquals(basicInfo.getQueryStats().getPeakUserMemoryReservation(), DataSize.valueOf("23GB")); assertEquals(basicInfo.getQueryStats().getTotalCpuTime(), Duration.valueOf("24m")); assertEquals(basicInfo.getQueryStats().isFullyBlocked(), true); assertEquals(basicInfo.getQueryStats().getBlockedReasons(), ImmutableSet.of(BlockedReason.WAITING_FOR_MEMORY)); assertEquals(basicInfo.getQueryStats().getProgressPercentage(), OptionalDouble.of(100));
public static QueryStateInfo createQueryStateInfo(BasicQueryInfo queryInfo, Optional<ResourceGroupId> group) { Optional<QueryProgressStats> progress = Optional.empty(); if (!queryInfo.getState().isDone() && queryInfo.getState() != QUEUED) { progress = Optional.of(createQueryProgressStats(queryInfo.getQueryStats())); } return createQueryStateInfo(queryInfo, group, Optional.empty(), progress); }
private static void logQueryTimeline(BasicQueryInfo queryInfo) { DateTime queryStartTime = queryInfo.getQueryStats().getCreateTime(); DateTime queryEndTime = queryInfo.getQueryStats().getEndTime(); // query didn't finish cleanly if (queryStartTime == null || queryEndTime == null) { return; } long elapsed = max(queryEndTime.getMillis() - queryStartTime.getMillis(), 0); logQueryTimeline( queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, elapsed, 0, 0, 0, queryStartTime, queryEndTime); }
public static void waitForQueryState(DistributedQueryRunner queryRunner, QueryId queryId, Set<QueryState> expectedQueryStates) throws InterruptedException { QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); do { // Heartbeat all the running queries, so they don't die while we're waiting for (BasicQueryInfo queryInfo : queryManager.getQueries()) { if (queryInfo.getState() == RUNNING) { queryManager.recordHeartbeat(queryInfo.getQueryId()); } } MILLISECONDS.sleep(500); } while (!expectedQueryStates.contains(queryManager.getQueryState(queryId))); }