@Override public void addStateChangeListener(QueryId queryId, StateChangeListener<QueryState> listener) { requireNonNull(listener, "listener is null"); queryTracker.getQuery(queryId).addStateChangeListener(listener); }
@Override public void cancelStage(StageId stageId) { requireNonNull(stageId, "stageId is null"); log.debug("Cancel stage %s", stageId); queryTracker.tryGetQuery(stageId.getQueryId()) .ifPresent(query -> query.cancelStage(stageId)); }
/** * Enforce query CPU time limits */ private void enforceCpuLimits() { for (QueryExecution query : queryTracker.getAllQueries()) { Duration cpuTime = query.getTotalCpuTime(); Duration sessionLimit = getQueryMaxCpuTime(query.getSession()); Duration limit = Ordering.natural().min(maxQueryCpuTime, sessionLimit); if (cpuTime.compareTo(limit) > 0) { query.fail(new ExceededCpuLimitException(limit)); } } }
private QueryMemoryInfo createQueryMemoryInfo(QueryExecution query) { // when the legacy system pool is enabled we use the user memory instead of the total memory if (isLegacySystemPoolEnabled) { return new QueryMemoryInfo(query.getQueryId(), query.getMemoryPool().getId(), query.getUserMemoryReservation().toBytes()); } return new QueryMemoryInfo(query.getQueryId(), query.getMemoryPool().getId(), query.getTotalMemoryReservation().toBytes()); }
private long getQueryMemoryReservation(QueryExecution query) { // when the legacy system pool is enabled we use the user memory instead of the total memory if (isLegacySystemPoolEnabled) { return query.getUserMemoryReservation().toBytes(); } return query.getTotalMemoryReservation().toBytes(); }
long maxMemory = -1; for (QueryExecution queryExecution : queries) { if (resourceOvercommit(queryExecution.getSession())) { log.info("Moving query %s to the reserved pool", biggestQuery.getQueryId()); biggestQuery.setMemoryPool(new VersionedMemoryPoolId(RESERVED_POOL, version)); assignments.add(new MemoryPoolAssignment(queryExecution.getQueryId(), queryExecution.getMemoryPool().getId()));
long totalMemoryBytes = 0L; for (QueryExecution query : runningQueries) { boolean resourceOvercommit = resourceOvercommit(query.getSession()); long userMemoryReservation = query.getUserMemoryReservation().toBytes(); long totalMemoryReservation = query.getTotalMemoryReservation().toBytes(); query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory))); queryKilled = true; long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes()); if (userMemoryReservation > userMemoryLimit) { query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit))); queryKilled = true; long totalMemoryLimit = min(maxQueryTotalMemory.toBytes(), getQueryMaxTotalMemory(query.getSession()).toBytes()); if (!isLegacySystemPoolEnabled && totalMemoryReservation > totalMemoryLimit) { query.fail(exceededGlobalTotalLimit(succinctBytes(totalMemoryLimit))); queryKilled = true; MemoryPoolId id = query.getMemoryPool().getId(); countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
long totalBytes = 0; for (QueryExecution query : queries) { long bytes = query.getTotalMemoryReservation(); DataSize sessionMaxQueryMemory = getQueryMaxMemory(query.getSession()); long queryMemoryLimit = Math.min(maxQueryMemory.toBytes(), sessionMaxQueryMemory.toBytes()); totalBytes += bytes; if (bytes > queryMemoryLimit) { if (resourceOvercommit(query.getSession())) { query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory, you set %s=true, and your query is using %s of memory, so it was killed.", RESOURCE_OVERCOMMIT, memory))); queryKilled = true; query.fail(exceededGlobalLimit(maxMemory)); queryKilled = true; long maxMemory = -1; for (QueryExecution query : queries) { long bytesUsed = query.getTotalMemoryReservation(); if (bytesUsed > maxMemory && query.getMemoryPool().getId().equals(GENERAL_POOL)) { biggestQuery = query; maxMemory = bytesUsed; biggestQuery.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "The cluster is out of memory, and your query was killed. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = biggestQuery.getQueryId(); MemoryPoolId id = query.getMemoryPool().getId(); countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
queryExecution.addStateChangeListener(newState -> { if (newState.isDone()) { queryExecution.getSession().getTransactionId().ifPresent(transactionManager::trySetInactive); queryTracker.addQuery(execution); BasicQueryInfo queryInfo = execution.getBasicQueryInfo(); queryMonitor.queryCreatedEvent(queryInfo); queryMonitor.queryImmediateFailureEvent(queryInfo, toFailure(e)); stats.queryStarted(); stats.queryStopped(); stats.queryFinished(execution.getQueryInfo()); queryMonitor.queryCreatedEvent(queryExecution.getBasicQueryInfo()); queryExecution.addFinalQueryInfoListener(finalQueryInfo -> { try { stats.queryFinished(finalQueryInfo);
/** * Enforce timeout at the query level */ public void enforceQueryMaxRunTimeLimits() { for (QueryExecution query : queries.values()) { if (query.getState().isDone()) { continue; } Duration queryMaxRunTime = SystemSessionProperties.getQueryMaxRunTime(query.getSession()); DateTime executionStartTime = query.getQueryInfo().getQueryStats().getCreateTime(); if (executionStartTime.plus(queryMaxRunTime.toMillis()).isBeforeNow()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded maximum time limit of " + queryMaxRunTime)); } } }
@PreDestroy public void stop() { boolean queryCancelled = false; for (QueryExecution queryExecution : queries.values()) { if (queryExecution.getState().isDone()) { continue; } log.info("Server shutting down. Query %s has been cancelled", queryExecution.getQueryId()); queryExecution.fail(new PrestoException(SERVER_SHUTTING_DOWN, "Server is shutting down. Query " + queryExecution.getQueryId() + " has been cancelled")); queryCancelled = true; } if (queryCancelled) { try { TimeUnit.SECONDS.sleep(5); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } queryManagementExecutor.shutdownNow(); queryExecutor.shutdownNow(); }
public void failAbandonedQueries() { for (QueryExecution queryExecution : queries.values()) { QueryInfo queryInfo = queryExecution.getQueryInfo(); if (queryInfo.getState().isDone()) { continue; } if (isAbandoned(queryExecution)) { log.info("Failing abandoned query %s", queryExecution.getQueryId()); queryExecution.fail(new AbandonedException("Query " + queryInfo.getQueryId(), queryInfo.getQueryStats().getLastHeartbeat(), DateTime.now())); } } }
QueryInfo queryInfo = execution.getQueryInfo(); queryMonitor.createdEvent(queryInfo); queryMonitor.completionEvent(queryInfo); QueryInfo queryInfo = queryExecution.getQueryInfo(); queryMonitor.createdEvent(queryInfo); queryExecution.addStateChangeListener(newValue -> { if (newValue.isDone()) { QueryInfo info = queryExecution.getQueryInfo(); queryExecution.fail(new PrestoException(QUERY_QUEUE_FULL, "Too many queued queries!"));
private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
@Override public QueryInfo getFullQueryInfo(QueryId queryId) { return queryTracker.getQuery(queryId).getQueryInfo(); }
/** * Set up a callback to fire when a query is completed. The callback will be called at most once. */ static void addCompletionCallback(QueryExecution queryExecution, Runnable callback) { AtomicBoolean taskExecuted = new AtomicBoolean(); queryExecution.addStateChangeListener(newValue -> { if (newValue.isDone() && taskExecuted.compareAndSet(false, true)) { callback.run(); } }); // Need to do this check in case the state changed before we added the previous state change listener if (queryExecution.getState().isDone() && taskExecuted.compareAndSet(false, true)) { callback.run(); } } }
@Override public QueryState getQueryState(QueryId queryId) { return queryTracker.getQuery(queryId).getState(); }
public void start() { // Only execute if the query is not already completed (e.g. cancelled) if (listenableFuture.isDone()) { return; } if (nextQueues.isEmpty()) { executor.execute(() -> { try (SetThreadName ignored = new SetThreadName("Query-%s", queryExecution.getQueryId())) { stats.queryStarted(); listenableFuture.addListener(stats::queryStopped, MoreExecutors.directExecutor()); queryExecution.start(); } }); } else { nextQueues.get(0).enqueue(new QueuedExecution(queryExecution, nextQueues.subList(1, nextQueues.size()), executor, stats, listenableFuture)); } } }
@Override public void failQuery(QueryId queryId, Throwable cause) { requireNonNull(cause, "cause is null"); queryTracker.tryGetQuery(queryId) .ifPresent(query -> query.fail(cause)); }
@Override public boolean submit(QueryExecution queryExecution, Executor executor, SqlQueryManagerStats stats) { List<QueryQueue> queues = selectQueues(queryExecution.getSession(), executor); for (QueryQueue queue : queues) { if (!queue.reserve(queryExecution)) { // Reject query if we couldn't acquire a permit to enter the queue. // The permits will be released when this query fails. return false; } } queues.get(0).enqueue(createQueuedExecution(queryExecution, queues.subList(1, queues.size()), executor, stats)); return true; }