InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator, OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators) { List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
/** * All the *active* primary shards for the provided indices grouped (each group is a single element, consisting * of the primary shard). This is handy for components that expect to get group iterators, but still want in some * cases to iterate over all primary shards (and not just one shard in replication group). * * @param indices The indices to return all the shards (replicas) * @return All the primary shards grouped into a single shard element group each * @throws IndexNotFoundException If an index passed does not exists */ public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { ShardRouting primary = indexShardRoutingTable.primaryShard(); if (primary.active()) { set.add(primary.shardsIt()); } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(primary.shardId(), Collections.<ShardRouting>emptyList())); } } } return new GroupShardsIterator<>(set); }
public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference, @Nullable ResponseCollectorService collectorService, @Nullable Map<String, Long> nodeCounts) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference, collectorService, nodeCounts); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator<>(new ArrayList<>(set)); }
/** * A groups shards iterator where each groups is a single {@link ShardRouting} and a group * is created for each shard routing. * <p> * This basically means that components that use the {@link GroupShardsIterator} will iterate * over *all* the shards (all the replicas) within the index.</p> */ public GroupShardsIterator groupByAllIt() { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (IndexShardRoutingTable indexShard : this) { for (ShardRouting shardRouting : indexShard) { set.add(shardRouting.shardsIt()); } } return new GroupShardsIterator(set); }
/** * A group shards iterator where each group ({@link ShardIterator} * is an iterator across shard replication group. */ public GroupShardsIterator groupByShardsIt() { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(shards.size()); for (IndexShardRoutingTable indexShard : this) { set.add(indexShard.shardsIt()); } return new GroupShardsIterator(set); }
static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator, OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators) { List<SearchShardIterator> shards = new ArrayList<>(); for (SearchShardIterator shardIterator : remoteShardIterators) { shards.add(shardIterator); } for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); }
InitialSearchPhase(String name, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts, Logger logger, int maxConcurrentShardRequests, Executor executor) { super(name); this.request = request; final List<SearchShardIterator> toSkipIterators = new ArrayList<>(); final List<SearchShardIterator> iterators = new ArrayList<>(); for (final SearchShardIterator iterator : shardsIts) { if (iterator.skip()) { toSkipIterators.add(iterator); } else { iterators.add(iterator); } } this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators); this.shardsIts = new GroupShardsIterator<>(iterators); this.logger = logger; // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up // it's number of active shards but use 1 as the default if no replica of a shard is active at this point. // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result // we process hence we add one for the non active partition here. this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty(); this.maxConcurrentShardRequests = Math.min(maxConcurrentShardRequests, shardsIts.size()); this.executor = executor; }
static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator, OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators) { List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); }
static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(GroupShardsIterator<ShardIterator> localShardsIterator, OriginalIndices localIndices, List<SearchShardIterator> remoteShardIterators) { List<SearchShardIterator> shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } return new GroupShardsIterator<>(shards); }
public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator<>(new ArrayList<>(set)); }
public GroupShardsIterator searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().localNodeId(), clusterState.nodes(), preference); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator(new ArrayList<>(set)); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
/** * All the *active* primary shards for the provided indices grouped (each group is a single element, consisting * of the primary shard). This is handy for components that expect to get group iterators, but still want in some * cases to iterate over all primary shards (and not just one shard in replication group). * * @param indices The indices to return all the shards (replicas) * @return All the primary shards grouped into a single shard element group each * @throws IndexNotFoundException If an index passed does not exists */ public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { ShardRouting primary = indexShardRoutingTable.primaryShard(); if (primary.active()) { set.add(primary.shardsIt()); } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(primary.shardId(), Collections.<ShardRouting>emptyList())); } } } return new GroupShardsIterator<>(set); }
/** * All the *active* primary shards for the provided indices grouped (each group is a single element, consisting * of the primary shard). This is handy for components that expect to get group iterators, but still want in some * cases to iterate over all primary shards (and not just one shard in replication group). * * @param indices The indices to return all the shards (replicas) * @return All the primary shards grouped into a single shard element group each * @throws IndexNotFoundException If an index passed does not exists */ public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { ShardRouting primary = indexShardRoutingTable.primaryShard(); if (primary.active()) { set.add(primary.shardsIt()); } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(primary.shardId(), Collections.<ShardRouting>emptyList())); } } } return new GroupShardsIterator<>(set); }
public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference, @Nullable ResponseCollectorService collectorService, @Nullable Map<String, Long> nodeCounts) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference, collectorService, nodeCounts); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator<>(new ArrayList<>(set)); }
public GroupShardsIterator<ShardIterator> searchShards(ClusterState clusterState, String[] concreteIndices, @Nullable Map<String, Set<String>> routing, @Nullable String preference, @Nullable ResponseCollectorService collectorService, @Nullable Map<String, Long> nodeCounts) { final Set<IndexShardRoutingTable> shards = computeTargetedShards(clusterState, concreteIndices, routing); final Set<ShardIterator> set = new HashSet<>(shards.size()); for (IndexShardRoutingTable shard : shards) { ShardIterator iterator = preferenceActiveShardIterator(shard, clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference, collectorService, nodeCounts); if (iterator != null) { set.add(iterator); } } return new GroupShardsIterator<>(new ArrayList<>(set)); }