/** * Selects the source shards for a local shard recovery. This might either be a split or a shrink operation. * @param shardId the target shard ID to select the source shards for * @param sourceIndexMetadata the source metadata * @param numTargetShards the number of target shards */ public static Set<ShardId> selectRecoverFromShards(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { if (sourceIndexMetadata.getNumberOfShards() > numTargetShards) { return selectShrinkShards(shardId, sourceIndexMetadata, numTargetShards); } else if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { return Collections.singleton(selectSplitShard(shardId, sourceIndexMetadata, numTargetShards)); } throw new IllegalArgumentException("can't select recover from shards if both indices have the same number of shards"); }
/** * Returns the source shard ids to shrink into the given shard id. * @param shardId the id of the target shard to shrink to * @param sourceIndexMetadata the source index metadata * @param numTargetShards the total number of shards in the target index * @return a set of shard IDs to shrink into the given shard ID. */ public static Set<ShardId> selectShrinkShards(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { if (shardId >= numTargetShards) { throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { throw new IllegalArgumentException("the number of target shards [" + numTargetShards +"] must be less that the number of source shards [" + sourceIndexMetadata.getNumberOfShards() + "]"); } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set<ShardId> shards = new HashSet<>(routingFactor); for (int i = shardId * routingFactor; i < routingFactor*shardId + routingFactor; i++) { shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); } return shards; }
private void validateExistingIndex(IndexMetaData currentIndexMetaData, IndexMetaData snapshotIndexMetaData, String renamedIndex, boolean partial) { // Index exist - checking that it's closed if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) { // TODO: Enable restore for open indices throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because an open index with same name already exists in the cluster. " + "Either close or delete the existing index or restore the index under a different name by providing a rename pattern and replacement name"); } // Index exist - checking if it's partial restore if (partial) { throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists"); } // Make sure that the number of shards is the same. That's the only thing that we cannot change if (currentIndexMetaData.getNumberOfShards() != snapshotIndexMetaData.getNumberOfShards()) { throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() + "] shards from a snapshot of index [" + snapshotIndexMetaData.getIndex().getName() + "] with [" + snapshotIndexMetaData.getNumberOfShards() + "] shards"); } }
private int getTotalNewShards(Index index, ClusterState currentState, int updatedNumberOfReplicas) { IndexMetaData indexMetaData = currentState.metaData().index(index); int shardsInIndex = indexMetaData.getNumberOfShards(); int oldNumberOfReplicas = indexMetaData.getNumberOfReplicas(); int replicaIncrease = updatedNumberOfReplicas - oldNumberOfReplicas; return replicaIncrease * shardsInIndex; }
IndexMetaData.selectShrinkShards(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); if (sourceMetaData.getNumberOfShards() == 1) { throw new IllegalArgumentException("can't shrink an index with only one shard"); int numShards = sourceMetaData.getNumberOfShards(); for (ShardRouting routing : table.shardsWithState(ShardRoutingState.STARTED)) { nodesToNumRouting.computeIfAbsent(routing.currentNodeId(), (s) -> new AtomicInteger(0)).incrementAndGet();
.append("], sv[").append(indexMetaData.getSettingsVersion()) .append("]\n"); for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) { sb.append(TAB).append(TAB).append(shard).append(": "); sb.append("p_term [").append(indexMetaData.primaryTerm(shard)).append("], ");
private static int getTotalShardCount(ClusterState state, Index index) { IndexMetaData indexMetaData = state.metaData().index(index); return indexMetaData.getNumberOfShards() * (1 + indexMetaData.getNumberOfReplicas()); }
/** * Returns the source shard ID to split the given target shard off * @param shardId the id of the target shard to split into * @param sourceIndexMetadata the source index metadata * @param numTargetShards the total number of shards in the target index * @return a the source shard ID to split off from */ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { if (shardId >= numTargetShards) { throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } int numSourceShards = sourceIndexMetadata.getNumberOfShards(); if (numSourceShards > numTargetShards) { throw new IllegalArgumentException("the number of source shards [" + numSourceShards + "] must be less that the number of target shards [" + numTargetShards + "]"); } int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); // now we verify that the numRoutingShards is valid in the source index int routingNumShards = sourceIndexMetadata.getRoutingNumShards(); if (routingNumShards % numTargetShards != 0) { throw new IllegalStateException("the number of routing shards [" + routingNumShards + "] must be a multiple of the target shards [" + numTargetShards + "]"); } // this is just an additional assertion that ensures we are a factor of the routing num shards. assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); }
for (ObjectCursor<IndexMetaData> cursor : indices.values()) { totalNumberOfShards += cursor.value.getTotalNumberOfShards(); numberOfShards += cursor.value.getNumberOfShards(); if (IndexMetaData.State.OPEN.equals(cursor.value.getState())) { totalOpenIndexShards += cursor.value.getTotalNumberOfShards();
static IndexMetaData validateResize(ClusterState state, String sourceIndex, Set<String> targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { if (state.metaData().hasIndex(targetIndexName)) { throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex()); } final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex); if (sourceMetaData == null) { throw new IndexNotFoundException(sourceIndex); } // ensure index is read-only if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\""); } if ((targetIndexMappingsTypes.size() > 1 || (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) { throw new IllegalArgumentException("mappings are not allowed when resizing indices" + ", all mappings are copied from the source index"); } if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { // this method applies all necessary checks ie. if the target shards are less than the source shards // of if the source shards are divisible by the number of target shards IndexMetaData.getRoutingFactor(sourceMetaData.getNumberOfShards(), IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } return sourceMetaData; }
final boolean isSplit = sourceMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); assert isSplit == false || sourceMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + "single type but the index is created before 6.0.0";
public ClusterIndexHealth(final IndexMetaData indexMetaData, final IndexRoutingTable indexRoutingTable) { this.index = indexMetaData.getIndex().getName(); this.numberOfShards = indexMetaData.getNumberOfShards(); this.numberOfReplicas = indexMetaData.getNumberOfReplicas();
for (Index index : concreteIndices) { final IndexMetaData indexMetaData = state.metaData().getIndexSafe(index); numberOfShards += indexMetaData.getNumberOfShards(); results.put(index.getName(), Collections.synchronizedList(new ArrayList<>())); final String index = concreteIndex.getName(); final IndexMetaData indexMetaData = state.metaData().getIndexSafe(concreteIndex); final int indexNumberOfShards = indexMetaData.getNumberOfShards(); for (int shard = 0; shard < indexNumberOfShards; shard++) { final ShardId shardId = new ShardId(indexMetaData.getIndex(), shard);
throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) { ShardId shardId = new ShardId(index, shardNumber); final RecoverySource primaryRecoverySource;
IndexMetaData indexMetaData = repository.getSnapshotIndexMetaData(snapshotInfo.snapshotId(), indexId); if (indexMetaData != null) { int numberOfShards = indexMetaData.getNumberOfShards(); for (int i = 0; i < numberOfShards; i++) { ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { builder.value(indexMetaData.primaryTerm(i));
/** * Initializes an index, to be restored from snapshot */ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecoverySource recoverySource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) { assert indexMetaData.getIndex().equals(index); if (!shards.isEmpty()) { throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) { ShardId shardId = new ShardId(index, shardNumber); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { boolean primary = i == 0; if (asNew && ignoreShards.contains(shardNumber)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); } else { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo)); } } shards.put(shardNumber, indexShardRoutingBuilder.build()); } return this; }
if (indexMetaData.getNumberOfShards() < sourceIndexMetaData.getNumberOfShards()) { ShardId shardId = IndexMetaData.selectSplitShard(shardRouting.id(), sourceIndexMetaData, indexMetaData.getNumberOfShards()); ShardRouting sourceShardRouting = allocation.routingNodes().activePrimary(shardId); if (sourceShardRouting == null) {
/** * Returns the expected shard size for the given shard or the default value provided if not enough information are available * to estimate the shards size. */ public static long getExpectedShardSize(ShardRouting shard, RoutingAllocation allocation, long defaultValue) { final IndexMetaData metaData = allocation.metaData().getIndexSafe(shard.index()); final ClusterInfo info = allocation.clusterInfo(); if (metaData.getResizeSourceIndex() != null && shard.active() == false && shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in // the worst case long targetShardSize = 0; final Index mergeSourceIndex = metaData.getResizeSourceIndex(); final IndexMetaData sourceIndexMeta = allocation.metaData().index(mergeSourceIndex); if (sourceIndexMeta != null) { final Set<ShardId> shardIds = IndexMetaData.selectRecoverFromShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) { if (shardIds.contains(shardRoutingTable.shardId())) { targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0); } } } return targetShardSize == 0 ? defaultValue : targetShardSize; } else { return info.getShardSize(shard, defaultValue); } } }
for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { ShardId shardId = new ShardId(indexMetaData.getIndex(), i); builder.put(shardId, new SnapshotsInProgress.ShardSnapshotStatus(null, State.MISSING, "index is closed")); for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { ShardId shardId = new ShardId(indexMetaData.getIndex(), i); if (indexRoutingTable != null) {