CanMatchPreFilterSearchPhase(Logger logger, SearchTransportService searchTransportService, BiFunction<String, String, Transport.Connection> nodeIdToConnection, Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts, Map<String, Set<String>> indexRoutings, Executor executor, SearchRequest request, ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts, TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory, SearchResponse.Clusters clusters) { //We set max concurrent shard requests to the number of shards so no throttling happens for can_match requests super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new BitSetSearchPhaseResults(shardsIts.size()), shardsIts.size(), clusters); this.phaseFactory = phaseFactory; this.shardsIts = shardsIts; }
SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) { super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) { super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
private void maybeExecuteNext() { final int index = shardExecutionIndex.getAndIncrement(); if (index < shardsIts.size()) { final SearchShardIterator shardRoutings = shardsIts.get(index); performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); } }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators) { SearchSourceBuilder source = searchRequest.source(); return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time SearchService.canRewriteToMatchNone(source) && searchRequest.getPreFilterShardSize() < shardIterators.size(); }
skipShard(iterator); if (shardsIts.size() > 0) { int maxConcurrentShardRequests = Math.min(this.maxConcurrentShardRequests, shardsIts.size()); final boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests); assert success; final StringBuilder missingShards = new StringBuilder(); for (int index = 0; index < shardsIts.size(); index++) { final SearchShardIterator shardRoutings = shardsIts.get(index); if (shardRoutings.size() == 0) {
(request.preference() != null || request.indexRoutings().length > 0)) { GroupShardsIterator<ShardIterator> group = buildShardIterator(clusterService, request); assert group.size() <= numShards : "index routing shards: " + group.size() + " cannot be greater than total number of shards: " + numShards; if (group.size() < numShards) { numShards = group.size(); int ord = 0; shardId = -1;
.searchShards(clusterState, concreteIndices, routingMap, request.preference()); ShardRouting shard; ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()]; int currentGroup = 0; for (ShardIterator shardIt : groupShardsIterator) {
SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, final long clusterStateVersion, final SearchTask task) { super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new SearchPhaseResults<>(shardsIts.size())); this.searchPhaseController = searchPhaseController; }
SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task) { super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size())); this.searchPhaseController = searchPhaseController; }
SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) { super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) { super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) { super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) { super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
private void maybeExecuteNext() { final int index = shardExecutionIndex.getAndIncrement(); if (index < shardsIts.size()) { final SearchShardIterator shardRoutings = shardsIts.get(index); performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); } }
private void maybeExecuteNext() { final int index = shardExecutionIndex.getAndIncrement(); if (index < shardsIts.size()) { final SearchShardIterator shardRoutings = shardsIts.get(index); performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull()); } }
private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators) { SearchSourceBuilder source = searchRequest.source(); return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time SearchService.canRewriteToMatchNone(source) && searchRequest.getPreFilterShardSize() < shardIterators.size(); }
private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators) { SearchSourceBuilder source = searchRequest.source(); return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time SearchService.canRewriteToMatchNone(source) && searchRequest.getPreFilterShardSize() < shardIterators.size(); }