private static Set<String> expand(Context context, IndexMetaData.State excludeState, Map<String, AliasOrIndex> matches) { Set<String> expand = new HashSet<>(); for (Map.Entry<String, AliasOrIndex> entry : matches.entrySet()) { AliasOrIndex aliasOrIndex = entry.getValue(); if (context.isPreserveAliases() && aliasOrIndex.isAlias()) { expand.add(entry.getKey()); } else { for (IndexMetaData meta : aliasOrIndex.getIndices()) { if (excludeState == null || meta.getState() != excludeState) { expand.add(meta.getIndex().getName()); } } } } return expand; }
/** * Elasticsearch v6.0 no longer supports indices created pre v5.0. All indices * that were created before Elasticsearch v5.0 should be re-indexed in Elasticsearch 5.x * before they can be opened by this version of elasticsearch. */ private void checkSupportedVersion(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) { if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion) == false) { throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created with version [" + indexMetaData.getCreationVersion() + "] but the minimum compatible version is [" + minimumIndexCompatibilityVersion + "]. It should be re-indexed in Elasticsearch " + minimumIndexCompatibilityVersion.major + ".x before upgrading to " + Version.CURRENT + "."); } }
/** * Validates whether a list of indices can be opened without going over the cluster shard limit. Only counts indices which are * currently closed and will be opened, ignores indices which are already open. * * @param currentState The current cluster state. * @param indices The indices which are to be opened. * @param deprecationLogger The logger to use to emit a deprecation warning, if appropriate. * @throws ValidationException If this operation would take the cluster over the limit and enforcement is enabled. */ static void validateShardLimit(ClusterState currentState, Index[] indices, DeprecationLogger deprecationLogger) { int shardsToOpen = Arrays.stream(indices) .filter(index -> currentState.metaData().index(index).getState().equals(IndexMetaData.State.CLOSE)) .mapToInt(index -> getTotalShardCount(currentState, index)) .sum(); Optional<String> error = IndicesService.checkShardLimit(shardsToOpen, currentState, deprecationLogger); if (error.isPresent()) { ValidationException ex = new ValidationException(); ex.addValidationError(error.get()); throw ex; } }
public Builder addAsRecovery(IndexMetaData indexMetaData) { if (indexMetaData.getState() == IndexMetaData.State.OPEN) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsRecovery(indexMetaData); add(indexRoutingBuilder); } return this; }
public Builder addAsNew(IndexMetaData indexMetaData) { if (indexMetaData.getState() == IndexMetaData.State.OPEN) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsNew(indexMetaData); add(indexRoutingBuilder); } return this; }
public Builder addAsFromDangling(IndexMetaData indexMetaData) { if (indexMetaData.getState() == IndexMetaData.State.OPEN) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsFromDangling(indexMetaData); add(indexRoutingBuilder); } return this; }
public Builder addAsFromCloseToOpen(IndexMetaData indexMetaData) { if (indexMetaData.getState() == IndexMetaData.State.OPEN) { IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex()) .initializeAsFromCloseToOpen(indexMetaData); add(indexRoutingBuilder); } return this; }
/** * Checks if the are replicas with the auto-expand feature that need to be adapted. * Returns a map of updates, which maps the indices to be updated to the desired number of replicas. * The map has the desired number of replicas as key and the indices to update as value, as this allows the result * of this method to be directly applied to RoutingTable.Builder#updateNumberOfReplicas. */ public static Map<Integer, List<String>> getAutoExpandReplicaChanges(MetaData metaData, DiscoveryNodes discoveryNodes) { // used for translating "all" to a number final int dataNodeCount = discoveryNodes.getDataNodes().size(); Map<Integer, List<String>> nrReplicasChanged = new HashMap<>(); for (final IndexMetaData indexMetaData : metaData) { if (indexMetaData.getState() != IndexMetaData.State.CLOSE) { AutoExpandReplicas autoExpandReplicas = SETTING.get(indexMetaData.getSettings()); autoExpandReplicas.getDesiredNumberOfReplicas(dataNodeCount).ifPresent(numberOfReplicas -> { if (numberOfReplicas != indexMetaData.getNumberOfReplicas()) { nrReplicasChanged.computeIfAbsent(numberOfReplicas, ArrayList::new).add(indexMetaData.getIndex().getName()); } }); } } return nrReplicasChanged; } }
public static Set<Index> getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set<Index> previouslyWrittenIndices) { RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (newRoutingNode == null) { throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state"); } Set<Index> indices = new HashSet<>(); for (ShardRouting routing : newRoutingNode) { indices.add(routing.index()); } // we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if // we have it written on disk previously for (IndexMetaData indexMetaData : state.metaData()) { boolean isOrWasClosed = indexMetaData.getState().equals(IndexMetaData.State.CLOSE); // if the index is open we might still have to write the state if it just transitioned from closed to open // so we have to check for that as well. IndexMetaData previousMetaData = previousState.metaData().index(indexMetaData.getIndex()); if (previousMetaData != null) { isOrWasClosed = isOrWasClosed || previousMetaData.getState().equals(IndexMetaData.State.CLOSE); } if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) { indices.add(indexMetaData.getIndex()); } } return indices; }
/** * Returns list of indices with missing shards, and list of indices that are closed * * @param shards list of shard statuses * @return list of failed and closed indices */ private Tuple<Set<String>, Set<String>> indicesWithMissingShards(ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards, MetaData metaData) { Set<String> missing = new HashSet<>(); Set<String> closed = new HashSet<>(); for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards) { if (entry.value.state() == State.MISSING) { if (metaData.hasIndex(entry.key.getIndex().getName()) && metaData.getIndexSafe(entry.key.getIndex()).getState() == IndexMetaData.State.CLOSE) { closed.add(entry.key.getIndex().getName()); } else { missing.add(entry.key.getIndex().getName()); } } } return new Tuple<>(missing, closed); }
totalNumberOfShards += cursor.value.getTotalNumberOfShards(); numberOfShards += cursor.value.getNumberOfShards(); if (IndexMetaData.State.OPEN.equals(cursor.value.getState())) { totalOpenIndexShards += cursor.value.getTotalNumberOfShards();
"have been deleted or the cluster must be new"; final AllocatedIndices.IndexRemovalReason reason = indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED; logger.debug("{} removing index, [{}]", index, reason); indicesService.removeIndex(index, reason, "removing index (no shards allocated)");
private boolean addFailureIfIndexIsUnavailable(DocWriteRequest request, int idx, final ConcreteIndices concreteIndices, final MetaData metaData) { IndexNotFoundException cannotCreate = indicesThatCannotBeCreated.get(request.index()); if (cannotCreate != null) { addFailure(request, idx, cannotCreate); return true; } Index concreteIndex = concreteIndices.getConcreteIndex(request.index()); if (concreteIndex == null) { try { concreteIndex = concreteIndices.resolveIfAbsent(request); } catch (IndexClosedException | IndexNotFoundException ex) { addFailure(request, idx, ex); return true; } } IndexMetaData indexMetaData = metaData.getIndexSafe(concreteIndex); if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { addFailure(request, idx, new IndexClosedException(concreteIndex)); return true; } return false; }
private void validateExistingIndex(IndexMetaData currentIndexMetaData, IndexMetaData snapshotIndexMetaData, String renamedIndex, boolean partial) { // Index exist - checking that it's closed if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) { // TODO: Enable restore for open indices throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because an open index with same name already exists in the cluster. " + "Either close or delete the existing index or restore the index under a different name by providing a rename pattern and replacement name"); } // Index exist - checking if it's partial restore if (partial) { throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists"); } // Make sure that the number of shards is the same. That's the only thing that we cannot change if (currentIndexMetaData.getNumberOfShards() != snapshotIndexMetaData.getNumberOfShards()) { throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() + "] shards from a snapshot of index [" + snapshotIndexMetaData.getIndex().getName() + "] with [" + snapshotIndexMetaData.getNumberOfShards() + "] shards"); } }
final IndexShardRoutingTable getShardRoutingTable(ShardId shardId, ClusterState state) { final IndexRoutingTable indexRoutingTable = state.routingTable().index(shardId.getIndexName()); if (indexRoutingTable == null) { IndexMetaData index = state.getMetaData().index(shardId.getIndex()); if (index != null && index.getState() == IndexMetaData.State.CLOSE) { throw new IndexClosedException(shardId.getIndex()); } throw new IndexNotFoundException(shardId.getIndexName()); } final IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(shardId.id()); if (shardRoutingTable == null) { throw new ShardNotFoundException(shardId); } return shardRoutingTable; }
if (indexRoutingTable == null && indexMetaData.getState() == IndexMetaData.State.CLOSE) {
for (Index index : indices) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMetaData.getState() != IndexMetaData.State.CLOSE) { indicesToClose.add(indexMetaData);
for (Index index : request.indices()) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMetaData.getState() != IndexMetaData.State.OPEN) { indicesToOpen.add(indexMetaData);
public Builder addBlocks(IndexMetaData indexMetaData) { String indexName = indexMetaData.getIndex().getName(); if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { addIndexBlock(indexName, MetaDataIndexStateService.INDEX_CLOSED_BLOCK); } if (IndexMetaData.INDEX_READ_ONLY_SETTING.get(indexMetaData.getSettings())) { addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_BLOCK); } if (IndexMetaData.INDEX_BLOCKS_READ_SETTING.get(indexMetaData.getSettings())) { addIndexBlock(indexName, IndexMetaData.INDEX_READ_BLOCK); } if (IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(indexMetaData.getSettings())) { addIndexBlock(indexName, IndexMetaData.INDEX_WRITE_BLOCK); } if (IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.get(indexMetaData.getSettings())) { addIndexBlock(indexName, IndexMetaData.INDEX_METADATA_BLOCK); } if (IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.get(indexMetaData.getSettings())) { addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK); } return this; }
} else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) { for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { ShardId shardId = new ShardId(indexMetaData.getIndex(), i);