@Override public void failQuery(QueryId queryId, Throwable cause) { requireNonNull(cause, "cause is null"); queryTracker.tryGetQuery(queryId) .ifPresent(query -> query.fail(cause)); }
private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
/** * Enforce query CPU time limits */ private void enforceCpuLimits() { for (QueryExecution query : queryTracker.getAllQueries()) { Duration cpuTime = query.getTotalCpuTime(); Duration sessionLimit = getQueryMaxCpuTime(query.getSession()); Duration limit = Ordering.natural().min(maxQueryCpuTime, sessionLimit); if (cpuTime.compareTo(limit) > 0) { query.fail(new ExceededCpuLimitException(limit)); } } }
query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory))); queryKilled = true; long userMemoryLimit = min(maxQueryMemory.toBytes(), getQueryMaxMemory(query.getSession()).toBytes()); if (userMemoryReservation > userMemoryLimit) { query.fail(exceededGlobalUserLimit(succinctBytes(userMemoryLimit))); queryKilled = true; query.fail(exceededGlobalTotalLimit(succinctBytes(totalMemoryLimit))); queryKilled = true;
@Override public void cancelQuery(QueryId queryId) { requireNonNull(queryId, "queryId is null"); log.debug("Cancel query %s", queryId); QueryExecution query = queries.get(queryId); if (query != null) { query.fail(new PrestoException(USER_CANCELED, "Query was canceled")); } }
@PreDestroy public void stop() { boolean queryCancelled = false; for (QueryExecution queryExecution : queries.values()) { if (queryExecution.getState().isDone()) { continue; } log.info("Server shutting down. Query %s has been cancelled", queryExecution.getQueryId()); queryExecution.fail(new PrestoException(SERVER_SHUTTING_DOWN, "Server is shutting down. Query " + queryExecution.getQueryId() + " has been cancelled")); queryCancelled = true; } if (queryCancelled) { try { TimeUnit.SECONDS.sleep(5); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } queryManagementExecutor.shutdownNow(); queryExecutor.shutdownNow(); }
public void failAbandonedQueries() { for (QueryExecution queryExecution : queries.values()) { QueryInfo queryInfo = queryExecution.getQueryInfo(); if (queryInfo.getState().isDone()) { continue; } if (isAbandoned(queryExecution)) { log.info("Failing abandoned query %s", queryExecution.getQueryId()); queryExecution.fail(new AbandonedException("Query " + queryInfo.getQueryId(), queryInfo.getQueryStats().getLastHeartbeat(), DateTime.now())); } } }
/** * Enforce timeout at the query level */ public void enforceQueryMaxRunTimeLimits() { for (QueryExecution query : queries.values()) { if (query.getState().isDone()) { continue; } Duration queryMaxRunTime = SystemSessionProperties.getQueryMaxRunTime(query.getSession()); DateTime executionStartTime = query.getQueryInfo().getQueryStats().getCreateTime(); if (executionStartTime.plus(queryMaxRunTime.toMillis()).isBeforeNow()) { query.fail(new PrestoException(EXCEEDED_TIME_LIMIT, "Query exceeded maximum time limit of " + queryMaxRunTime)); } } }
queryExecution.fail(new PrestoException(QUERY_QUEUE_FULL, "Too many queued queries!"));
query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory, you set %s=true, and your query is using %s of memory, so it was killed.", RESOURCE_OVERCOMMIT, memory))); queryKilled = true; query.fail(exceededGlobalLimit(maxMemory)); queryKilled = true; biggestQuery.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "The cluster is out of memory, and your query was killed. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = biggestQuery.getQueryId();