@GET @Path("memory") public Response getClusterMemoryPoolInfo() { return Response.ok() .entity(clusterMemoryManager.getMemoryPoolInfo()) .build(); }
@GET @Path("workerMemory") public Response getWorkerMemoryInfo() { return Response.ok() .entity(clusterMemoryManager.getWorkerMemoryInfo()) .build(); }
boolean outOfMemory = isClusterOutOfMemory(); if (!outOfMemory) { lastTimeNotOutOfMemory = System.nanoTime(); DataSize memory = succinctBytes(getQueryMemoryReservation(query)); query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory))); !queryKilled && nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0) { if (isLastKilledQueryGone()) { callOomKiller(runningQueries); updatePools(countByPool); assignmentsRequest = updateAssignments(runningQueries); updateNodes(assignmentsRequest);
if (allAssignmentsHavePropagated(queries)) { if (reservedPool.getAssignedQueries() == 0 && generalPool.getBlockedNodes() > 0) { QueryExecution biggestQuery = null; long bytesUsed = getQueryMemoryReservation(queryExecution); if (bytesUsed > maxMemory) { biggestQuery = queryExecution;
private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
"maxQueryMemory cannot be greater than maxQueryTotalMemory"); this.pools = createClusterMemoryPools(nodeMemoryConfig.isReservedPoolEnabled());
boolean outOfMemory = isClusterOutOfMemory(); if (!outOfMemory) { lastTimeNotOutOfMemory = System.nanoTime(); DataSize memory = succinctBytes(getQueryMemoryReservation(query)); query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory))); !queryKilled && nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0) { if (isLastKilledQueryGone()) { callOomKiller(runningQueries); updatePools(countByPool); assignmentsRequest = updateAssignments(runningQueries); updateNodes(assignmentsRequest);
if (allAssignmentsHavePropagated(queries)) { if (reservedPool.getAssignedQueries() == 0 && generalPool.getBlockedNodes() > 0) { QueryExecution biggestQuery = null; long bytesUsed = getQueryMemoryReservation(queryExecution); if (bytesUsed > maxMemory) { biggestQuery = queryExecution;
private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
"maxQueryMemory cannot be greater than maxQueryTotalMemory"); this.pools = createClusterMemoryPools(nodeMemoryConfig.isLegacySystemPoolEnabled(), nodeMemoryConfig.isReservedPoolEnabled());
@GET @Path("memory") public Response getClusterMemoryPoolInfo() { return Response.ok() .entity(clusterMemoryManager.getMemoryPoolInfo()) .build(); }
@GET @Path("workerMemory") public Response getWorkerMemoryInfo() { return Response.ok() .entity(clusterMemoryManager.getWorkerMemoryInfo()) .build(); }