@Managed public int getNumberOfLeakedQueries() { return memoryLeakDetector.getNumberOfLeakedQueries(); }
/** * @param queryInfoSupplier All queries that the coordinator knows about. * @param queryMemoryReservations The memory reservations of queries in the GENERAL cluster memory pool. */ void checkForMemoryLeaks(Supplier<List<BasicQueryInfo>> queryInfoSupplier, Map<QueryId, Long> queryMemoryReservations) { requireNonNull(queryInfoSupplier); requireNonNull(queryMemoryReservations); Map<QueryId, BasicQueryInfo> queryIdToInfo = Maps.uniqueIndex(queryInfoSupplier.get(), BasicQueryInfo::getQueryId); Map<QueryId, Long> leakedQueryReservations = queryMemoryReservations.entrySet() .stream() .filter(entry -> entry.getValue() > 0) .filter(entry -> isLeaked(queryIdToInfo, entry.getKey())) .collect(toImmutableMap(Entry::getKey, Entry::getValue)); if (!leakedQueryReservations.isEmpty()) { log.debug("Memory leak detected. The following queries are already finished, " + "but they have memory reservations on some worker node(s): %s", leakedQueryReservations); } synchronized (this) { leakedQueries = ImmutableSet.copyOf(leakedQueryReservations.keySet()); } }
@GuardedBy("this") private boolean isLastKilledQueryGone() { if (lastKilledQuery == null) { return true; } // If the lastKilledQuery is marked as leaked by the ClusterMemoryLeakDetector we consider the lastKilledQuery as gone, // so that the ClusterMemoryManager can continue to make progress even if there are leaks. // Even if the weak references to the leaked queries are GCed in the ClusterMemoryLeakDetector, it will mark the same queries // as leaked in its next run, and eventually the ClusterMemoryManager will make progress. if (memoryLeakDetector.wasQueryPossiblyLeaked(lastKilledQuery)) { lastKilledQuery = null; return true; } // pools fields is updated based on nodes field. // Therefore, if the query is gone from pools field, it should also be gone from nodes field. // However, since nodes can updated asynchronously, it has the potential of coming back after being gone. // Therefore, even if the query appears to be gone here, it might be back when one inspects nodes later. return !pools.get(GENERAL_POOL) .getQueryMemoryReservations() .containsKey(lastKilledQuery); }
@Test public void testLeakDetector() { QueryId testQuery = new QueryId("test"); ClusterMemoryLeakDetector leakDetector = new ClusterMemoryLeakDetector(); leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(), ImmutableMap.of()); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report no leaked queries as the query is still running leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), RUNNING)), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report exactly one leaked query since the query is finished, and its end time is way in the past leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 1); // the leak detector should report no leaked queries as the query doesn't have any memory reservation leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 0L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 0); // the leak detector should report exactly one leaked query since the coordinator doesn't know of any query leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(), ImmutableMap.of(testQuery, 1L)); assertEquals(leakDetector.getNumberOfLeakedQueries(), 1); }
memoryLeakDetector.checkForMemoryLeaks(allQueryInfoSupplier, pools.get(GENERAL_POOL).getQueryMemoryReservations());
memoryLeakDetector.checkForMemoryLeaks(allQueryInfoSupplier, pools.get(GENERAL_POOL).getQueryMemoryReservations());
@Managed public int getNumberOfLeakedQueries() { return memoryLeakDetector.getNumberOfLeakedQueries(); }
@GuardedBy("this") private boolean isLastKilledQueryGone() { if (lastKilledQuery == null) { return true; } // If the lastKilledQuery is marked as leaked by the ClusterMemoryLeakDetector we consider the lastKilledQuery as gone, // so that the ClusterMemoryManager can continue to make progress even if there are leaks. // Even if the weak references to the leaked queries are GCed in the ClusterMemoryLeakDetector, it will mark the same queries // as leaked in its next run, and eventually the ClusterMemoryManager will make progress. if (memoryLeakDetector.wasQueryPossiblyLeaked(lastKilledQuery)) { lastKilledQuery = null; return true; } // pools fields is updated based on nodes field. // Therefore, if the query is gone from pools field, it should also be gone from nodes field. // However, since nodes can updated asynchronously, it has the potential of coming back after being gone. // Therefore, even if the query appears to be gone here, it might be back when one inspects nodes later. return !pools.get(GENERAL_POOL) .getQueryMemoryReservations() .containsKey(lastKilledQuery); }
/** * @param queryInfoSupplier All queries that the coordinator knows about. * @param queryMemoryReservations The memory reservations of queries in the GENERAL cluster memory pool. */ void checkForMemoryLeaks(Supplier<List<BasicQueryInfo>> queryInfoSupplier, Map<QueryId, Long> queryMemoryReservations) { requireNonNull(queryInfoSupplier); requireNonNull(queryMemoryReservations); Map<QueryId, BasicQueryInfo> queryIdToInfo = Maps.uniqueIndex(queryInfoSupplier.get(), BasicQueryInfo::getQueryId); Map<QueryId, Long> leakedQueryReservations = queryMemoryReservations.entrySet() .stream() .filter(entry -> entry.getValue() > 0) .filter(entry -> isLeaked(queryIdToInfo, entry.getKey())) .collect(toImmutableMap(Entry::getKey, Entry::getValue)); if (!leakedQueryReservations.isEmpty()) { log.debug("Memory leak detected. The following queries are already finished, " + "but they have memory reservations on some worker node(s): %s", leakedQueryReservations); } synchronized (this) { leakedQueries = ImmutableSet.copyOf(leakedQueryReservations.keySet()); } }