CanMatchPreFilterSearchPhase(Logger logger, SearchTransportService searchTransportService, BiFunction<String, String, Transport.Connection> nodeIdToConnection, Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts, Map<String, Set<String>> indexRoutings, Executor executor, SearchRequest request, ActionListener<SearchResponse> listener, GroupShardsIterator<SearchShardIterator> shardsIts, TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, Function<GroupShardsIterator<SearchShardIterator>, SearchPhase> phaseFactory, SearchResponse.Clusters clusters) { //We set max concurrent shard requests to the number of shards so no throttling happens for can_match requests super("can_match", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new BitSetSearchPhaseResults(shardsIts.size()), shardsIts.size(), clusters); this.phaseFactory = phaseFactory; this.shardsIts = shardsIts; }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator, OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators) { List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); }
protected AbstractSearchAsyncAction(ESLogger logger, SearchServiceTransportAction searchService, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, SearchPhaseController searchPhaseController, ThreadPool threadPool, SearchRequest request, ActionListener<SearchResponse> listener) { this.logger = logger; this.searchService = searchService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.searchPhaseController = searchPhaseController; this.threadPool = threadPool; this.request = request; this.listener = listener; this.clusterState = clusterService.state(); nodes = clusterState.nodes(); clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead // of just for the _search api String[] concreteIndices = indexNameExpressionResolver.concreteIndices(clusterState, request.indicesOptions(), startTime(), request.indices()); for (String index : concreteIndices) { clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index); } Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, request.routing(), request.indices()); shardsIts = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference()); expectedSuccessfulOps = shardsIts.size(); // we need to add 1 for non active partition, since we count it in the total! expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); firstResults = new AtomicArray<>(shardsIts.size()); }
@Override protected ShardIterator shards(ClusterState state, InternalRequest request) { if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting().searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); return groupShardsIter.iterator().next(); } return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger) { super(name); this.request = request; this.shardsIts = shardsIts; this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
@Override protected ShardIterator shards(ClusterState state, InternalRequest request) { if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting().searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); return groupShardsIter.iterator().next(); } return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); }
SearchQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, long clusterStateVersion, SearchTask task, SearchResponse.Clusters clusters) { super("query", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, searchPhaseController.newSearchPhaseResults(request, shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
/** * All the *active* primary shards for the provided indices grouped (each group is a single element, consisting * of the primary shard). This is handy for components that expect to get group iterators, but still want in some * cases to iterate over all primary shards (and not just one shard in replication group). * * @param indices The indices to return all the shards (replicas) * @return All the primary shards grouped into a single shard element group each * @throws IndexNotFoundException If an index passed does not exists */ public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { ShardRouting primary = indexShardRoutingTable.primaryShard(); if (primary.active()) { set.add(primary.shardsIt()); } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(primary.shardId(), Collections.<ShardRouting>emptyList())); } } } return new GroupShardsIterator<>(set); }
skipShard(iterator); if (shardsIts.size() > 0) { int maxConcurrentShardRequests = Math.min(this.maxConcurrentShardRequests, shardsIts.size()); final boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests); assert success; final StringBuilder missingShards = new StringBuilder(); for (int index = 0; index < shardsIts.size(); index++) { final SearchShardIterator shardRoutings = shardsIts.get(index); if (shardRoutings.size() == 0) { if(missingShards.length() >0 ){ final SearchShardIterator shardRoutings = shardsIts.get(index); assert shardRoutings.skip() == false; performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull());
@Override protected ShardIterator shards(ClusterState state, InternalRequest request) { if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting().searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); return groupShardsIter.iterator().next(); } return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); }
SearchDfsQueryThenFetchAsyncAction(final Logger logger, final SearchTransportService searchTransportService, final BiFunction<String, String, Transport.Connection> nodeIdToConnection, final Map<String, AliasFilter> aliasFilter, final Map<String, Float> concreteIndexBoosts, final Map<String, Set<String>> indexRoutings, final SearchPhaseController searchPhaseController, final Executor executor, final SearchRequest request, final ActionListener<SearchResponse> listener, final GroupShardsIterator<SearchShardIterator> shardsIts, final TransportSearchAction.SearchTimeProvider timeProvider, final long clusterStateVersion, final SearchTask task, SearchResponse.Clusters clusters) { super("dfs", logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, indexRoutings, executor, request, listener, shardsIts, timeProvider, clusterStateVersion, task, new ArraySearchPhaseResults<>(shardsIts.size()), request.getMaxConcurrentShardRequests(), clusters); this.searchPhaseController = searchPhaseController; }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference, @Nullable ResponseCollectorService collectorService, @Nullable Map<String, Long> nodeCounts) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference, collectorService, nodeCounts); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator<>(new ArrayList<>(set)); }
@Override protected ShardIterator shards(ClusterState state, InternalRequest request) { if (request.request().doc() != null && request.request().routing() == null) { // artificial document without routing specified, ignore its "id" and use either random shard or according to preference GroupShardsIterator<ShardIterator> groupShardsIter = clusterService.operationRouting().searchShards(state, new String[] { request.concreteIndex() }, null, request.request().preference()); return groupShardsIter.iterator().next(); } return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()); }
private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators) { SearchSourceBuilder source = searchRequest.source(); return searchRequest.searchType() == QUERY_THEN_FETCH && // we can't do this for DFS it needs to fan out to all shards all the time SearchService.canRewriteToMatchNone(source) && searchRequest.getPreFilterShardSize() < shardIterators.size(); }