/** * All the shards (replicas) for the provided index. * * @param index The index to return all the shards (replicas). * @return All the shards matching the specific index * @throws IndexNotFoundException If the index passed does not exists */ public List<ShardRouting> allShards(String index) { List<ShardRouting> shards = new ArrayList<>(); IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { shards.add(shardRouting); } } return shards; }
/** * Finds all indices that have not all primaries available */ private Set<String> indicesWithMissingPrimaries(ClusterState clusterState, String[] concreteIndices) { Set<String> indices = new HashSet<>(); RoutingTable routingTable = clusterState.routingTable(); for (String index : concreteIndices) { IndexRoutingTable indexRoutingTable = routingTable.index(index); if (indexRoutingTable.allPrimaryShardsActive() == false) { indices.add(index); } } return indices; }
public IndexShardRoutingTable shardRoutingTableOrNull(ShardId shardId) { return Optional .ofNullable(index(shardId.getIndexName())) .flatMap(irt -> Optional.ofNullable(irt.shard(shardId.getId()))) .orElse(null); }
protected IndexRoutingTable indexRoutingTable(ClusterState clusterState, String index) { IndexRoutingTable indexRouting = clusterState.routingTable().index(index); if (indexRouting == null) { throw new IndexNotFoundException(index); } return indexRouting; }
private ShardsIterator allShardsSatisfyingPredicate(String[] indices, Predicate<ShardRouting> predicate, boolean includeRelocationTargets) { // use list here since we need to maintain identity across shards List<ShardRouting> shards = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { shards.add(shardRouting); if (includeRelocationTargets && shardRouting.relocating()) { shards.add(shardRouting.getTargetRelocatingShard()); } } } } } return new PlainShardsIterator(shards); }
@Override protected ShardsIterator shards(ClusterState state, InternalRequest request) { // Will balance requests between shards // Resolve patterns and deduplicate return state.routingTable().index(request.concreteIndex()).randomAllActiveShardsIt(); }
@Override protected ShardsIterator shards(ClusterState state, InternalRequest request) { // Will balance requests between shards return state.routingTable().index(request.concreteIndex()).randomAllActiveShardsIt(); }
public boolean hasIndex(Index index) { IndexRoutingTable indexRouting = index(index.getName()); return indexRouting != null && indexRouting.getIndex().equals(index); }
private GroupShardsIterator<ShardIterator> allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate<ShardRouting> predicate) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { continue; // we simply ignore indices that don't exists (make sense for operations that use it currently) } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { for (ShardRouting shardRouting : indexShardRoutingTable) { if (predicate.test(shardRouting)) { set.add(shardRouting.shardsIt()); if (includeRelocationTargets && shardRouting.relocating()) { set.add(new PlainShardIterator(shardRouting.shardId(), Collections.singletonList(shardRouting.getTargetRelocatingShard()))); } } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(shardRouting.shardId(), Collections.<ShardRouting>emptyList())); } } } } return new GroupShardsIterator<>(set); }
@Override protected ShardsIterator shards(ClusterState state, InternalRequest request) { if (request.concreteIndex() == null) { // just execute locally.... return null; } return state.routingTable().index(request.concreteIndex()).randomAllActiveShardsIt(); }
/** * All the *active* primary shards for the provided indices grouped (each group is a single element, consisting * of the primary shard). This is handy for components that expect to get group iterators, but still want in some * cases to iterate over all primary shards (and not just one shard in replication group). * * @param indices The indices to return all the shards (replicas) * @return All the primary shards grouped into a single shard element group each * @throws IndexNotFoundException If an index passed does not exists */ public GroupShardsIterator<ShardIterator> activePrimaryShardsGrouped(String[] indices, boolean includeEmpty) { // use list here since we need to maintain identity across shards ArrayList<ShardIterator> set = new ArrayList<>(); for (String index : indices) { IndexRoutingTable indexRoutingTable = index(index); if (indexRoutingTable == null) { throw new IndexNotFoundException(index); } for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { ShardRouting primary = indexShardRoutingTable.primaryShard(); if (primary.active()) { set.add(primary.shardsIt()); } else if (includeEmpty) { // we need this for counting properly, just make it an empty one set.add(new PlainShardIterator(primary.shardId(), Collections.<ShardRouting>emptyList())); } } } return new GroupShardsIterator<>(set); }
/** * Returns <code>true</code> iff the routing table has changed for the given index. * Note that this is an object reference equality test, not an equals test. */ public boolean indexRoutingTableChanged(String index) { Objects.requireNonNull(index, "index must not be null"); if (!state.routingTable().hasIndex(index) && !previousState.routingTable().hasIndex(index)) { return false; } if (state.routingTable().hasIndex(index) && previousState.routingTable().hasIndex(index)) { return state.routingTable().index(index) != previousState.routingTable().index(index); } return true; }
/** * All shards for the provided index and shard id * @return All the shard routing entries for the given index and shard id * @throws IndexNotFoundException if provided index does not exist * @throws ShardNotFoundException if provided shard id is unknown */ public IndexShardRoutingTable shardRoutingTable(String index, int shardId) { IndexRoutingTable indexRouting = index(index); if (indexRouting == null) { throw new IndexNotFoundException(index); } IndexShardRoutingTable shard = indexRouting.shard(shardId); if (shard == null) { throw new ShardNotFoundException(new ShardId(indexRouting.getIndex(), shardId)); } return shard; }
@Override protected void masterOperation(IndicesShardStoresRequest request, ClusterState state, ActionListener<IndicesShardStoresResponse> listener) { final RoutingTable routingTables = state.routingTable(); final RoutingNodes routingNodes = state.getRoutingNodes(); final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); final Set<ShardId> shardIdsToFetch = new HashSet<>(); logger.trace("using cluster state version [{}] to determine shards", state.version()); // collect relevant shard ids of the requested indices for fetching store infos for (String index : concreteIndices) { IndexRoutingTable indexShardRoutingTables = routingTables.index(index); if (indexShardRoutingTables == null) { continue; } for (IndexShardRoutingTable routing : indexShardRoutingTables) { final int shardId = routing.shardId().id(); ClusterShardHealth shardHealth = new ClusterShardHealth(shardId, routing); if (request.shardStatuses().contains(shardHealth.getStatus())) { shardIdsToFetch.add(routing.shardId()); } } } // async fetch store infos from all the nodes // NOTE: instead of fetching shard store info one by one from every node (nShards * nNodes requests) // we could fetch all shard store info from every node once (nNodes requests) // we have to implement a TransportNodesAction instead of using TransportNodesListGatewayStartedShards // for fetching shard stores info, that operates on a list of shards instead of a single shard new AsyncShardStoresInfoFetches(state.nodes(), routingNodes, shardIdsToFetch, listener).start(); }
/** * Validate the name for an index against some static rules and a cluster state. */ public static void validateIndexName(String index, ClusterState state) { validateIndexOrAliasName(index, InvalidIndexNameException::new); if (!index.toLowerCase(Locale.ROOT).equals(index)) { throw new InvalidIndexNameException(index, "must be lowercase"); } if (state.routingTable().hasIndex(index)) { throw new ResourceAlreadyExistsException(state.routingTable().index(index).getIndex()); } if (state.metaData().hasIndex(index)) { throw new ResourceAlreadyExistsException(state.metaData().index(index).getIndex()); } if (state.metaData().hasAlias(index)) { throw new InvalidIndexNameException(index, "already exists as alias"); } }
@Override protected ShardIterator shards(ClusterState clusterState, UpdateRequest request) { if (request.getShardId() != null) { return clusterState.routingTable().index(request.concreteIndex()).shard(request.getShardId().getId()).primaryShardIt(); } ShardIterator shardIterator = clusterService.operationRouting() .indexShards(clusterState, request.concreteIndex(), request.id(), request.routing()); ShardRouting shard; while ((shard = shardIterator.nextOrNull()) != null) { if (shard.primary()) { return new PlainShardIterator(shardIterator.shardId(), Collections.singletonList(shard)); } } return new PlainShardIterator(shardIterator.shardId(), Collections.emptyList()); }
/** * All shards for the provided {@link ShardId} * @return All the shard routing entries for the given index and shard id * @throws IndexNotFoundException if provided index does not exist * @throws ShardNotFoundException if provided shard id is unknown */ public IndexShardRoutingTable shardRoutingTable(ShardId shardId) { IndexRoutingTable indexRouting = index(shardId.getIndexName()); if (indexRouting == null || indexRouting.getIndex().equals(shardId.getIndex()) == false) { throw new IndexNotFoundException(shardId.getIndex()); } IndexShardRoutingTable shard = indexRouting.shard(shardId.id()); if (shard == null) { throw new ShardNotFoundException(shardId); } return shard; }
final IndexShardRoutingTable getShardRoutingTable(ShardId shardId, ClusterState state) { final IndexRoutingTable indexRoutingTable = state.routingTable().index(shardId.getIndexName()); if (indexRoutingTable == null) { IndexMetaData index = state.getMetaData().index(shardId.getIndex()); if (index != null && index.getState() == IndexMetaData.State.CLOSE) { throw new IndexClosedException(shardId.getIndex()); } throw new IndexNotFoundException(shardId.getIndexName()); } final IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(shardId.id()); if (shardRoutingTable == null) { throw new ShardNotFoundException(shardId); } return shardRoutingTable; }
/** * Returns the expected shard size for the given shard or the default value provided if not enough information are available * to estimate the shards size. */ public static long getExpectedShardSize(ShardRouting shard, RoutingAllocation allocation, long defaultValue) { final IndexMetaData metaData = allocation.metaData().getIndexSafe(shard.index()); final ClusterInfo info = allocation.clusterInfo(); if (metaData.getResizeSourceIndex() != null && shard.active() == false && shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in // the worst case long targetShardSize = 0; final Index mergeSourceIndex = metaData.getResizeSourceIndex(); final IndexMetaData sourceIndexMeta = allocation.metaData().index(mergeSourceIndex); if (sourceIndexMeta != null) { final Set<ShardId> shardIds = IndexMetaData.selectRecoverFromShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) { if (shardIds.contains(shardRoutingTable.shardId())) { targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0); } } } return targetShardSize == 0 ? defaultValue : targetShardSize; } else { return info.getShardSize(shard, defaultValue); } } }
private boolean waitingShardsStartedOrUnassigned(ClusterChangedEvent event) { SnapshotsInProgress curr = event.state().custom(SnapshotsInProgress.TYPE); if (curr != null) { for (SnapshotsInProgress.Entry entry : curr.entries()) { if (entry.state() == State.STARTED && !entry.waitingIndices().isEmpty()) { for (ObjectCursor<String> index : entry.waitingIndices().keys()) { if (event.indexRoutingTableChanged(index.value)) { IndexRoutingTable indexShardRoutingTable = event.state().getRoutingTable().index(index.value); for (ShardId shardId : entry.waitingIndices().get(index.value)) { ShardRouting shardRouting = indexShardRoutingTable.shard(shardId.id()).primaryShard(); if (shardRouting != null && (shardRouting.started() || shardRouting.unassigned())) { return true; } } } } } } } return false; }