private synchronized void callOomKiller(Iterable<QueryExecution> runningQueries) { List<QueryMemoryInfo> queryMemoryInfoList = Streams.stream(runningQueries) .map(this::createQueryMemoryInfo) .collect(toImmutableList()); List<MemoryInfo> nodeMemoryInfos = nodes.values().stream() .map(RemoteNodeMemory::getInfo) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); Optional<QueryId> chosenQueryId = lowMemoryKiller.chooseQueryToKill(queryMemoryInfoList, nodeMemoryInfos); if (chosenQueryId.isPresent()) { log.debug("Low memory killer chose %s", chosenQueryId.get()); Optional<QueryExecution> chosenQuery = Streams.stream(runningQueries).filter(query -> chosenQueryId.get().equals(query.getQueryId())).collect(toOptional()); if (chosenQuery.isPresent()) { // See comments in isLastKilledQueryGone for why chosenQuery might be absent. chosenQuery.get().fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "Query killed because the cluster is out of memory. Please try again in a few minutes.")); queriesKilledDueToOutOfMemory.incrementAndGet(); lastKilledQuery = chosenQueryId.get(); logQueryKill(chosenQueryId.get(), nodeMemoryInfos); } } }
@Test public void testGeneralPoolNotBlocked() { int reservePool = 10; int generalPool = 12; Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 6L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_2", ImmutableMap.of("n1", 3L, "n2", 5L, "n3", 2L, "n4", 4L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.empty()); }
@Test public void testGeneralPoolHasNoReservation() { int reservePool = 10; int generalPool = 12; Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.empty()); }
@Test public void testGeneralPoolHasNoReservation() { int reservePool = 10; int generalPool = 12; Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.empty()); }
@Test public void testSkewedQuery() { int reservePool = 10; int generalPool = 12; // q2 is the query with the most total memory reservation, but not the query with the max memory reservation. // This also tests the corner case where a node doesn't have a general pool. Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 8L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_2", ImmutableMap.of("n1", 3L, "n2", 5L, "n3", 2L, "n4", 4L, "n5", 0L)) .put("q_3", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 9L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.of(new QueryId("q_2"))); } }
@Test public void testSkewedQuery() { int reservePool = 10; int generalPool = 12; // q1 is neither the query with the most total memory reservation, nor the query with the max memory reservation. // This also tests the corner case where a node doesn't have a general pool. Map<String, Map<String, Long>> queries = ImmutableMap.<String, Map<String, Long>>builder() .put("q_1", ImmutableMap.of("n1", 0L, "n2", 8L, "n3", 0L, "n4", 0L, "n5", 0L)) .put("q_2", ImmutableMap.of("n1", 3L, "n2", 5L, "n3", 2L, "n4", 4L, "n5", 0L)) .put("q_3", ImmutableMap.of("n1", 0L, "n2", 0L, "n3", 9L, "n4", 0L, "n5", 0L)) .put("q_r", ImmutableMap.of("n1", 6L, "n2", 6L, "n3", 6L, "n4", 6L, "n5", 6L)) .build(); assertEquals( lowMemoryKiller.chooseQueryToKill( toQueryMemoryInfoList("q_r", queries), toNodeMemoryInfoList(reservePool, generalPool, "q_r", queries)), Optional.of(new QueryId("q_1"))); } }