threadContext.putHeader(ConfigConstants.SG_CONF_REQUEST_HEADER, "true"); boolean searchGuardIndexExists = clusterService.state().metaData().hasConcreteIndex(this.searchguardIndex); if(clusterService.state().metaData().index(this.searchguardIndex).mapping("config") != null) { LOGGER.debug("sg index exists and was created before ES 6 (legacy layout)"); retVal.putAll(validate(legacycl.loadLegacy(configTypes.toArray(new String[0]), 5, TimeUnit.SECONDS), configTypes.size())); } else { LOGGER.debug("sg index exists and was created with ES 6 (new layout)"); retVal.putAll(validate(cl.load(configTypes.toArray(new String[0]), 5, TimeUnit.SECONDS), configTypes.size())); LOGGER.debug("sg index not exists (yet)"); retVal.putAll(validate(cl.load(configTypes.toArray(new String[0]), 30, TimeUnit.SECONDS), configTypes.size()));
final IndexMetaData indexMetaData = clusterService.state().metaData().getIndices().get(requestAliasOrIndex); log.debug("{} does not exist in cluster metadata", requestAliasOrIndex); continue; final ImmutableOpenMap<String, AliasMetaData> aliases = indexMetaData.getAliases(); if(aliases != null && aliases.size() > 0) { if(log.isDebugEnabled()) { log.debug("Aliases for {}: {}", requestAliasOrIndex, aliases); final Iterator<String> it = aliases.keysIt(); while(it.hasNext()) { final String alias = it.next();
public boolean equalsAliases(MetaData other) { for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) { IndexMetaData otherIndex = cursor.value; IndexMetaData thisIndex = index(otherIndex.getIndex()); if (thisIndex == null) { return false; } if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) { return false; } } return true; }
public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException { String[] nodesIds = clusterService.state().nodes().getMasterNodes().keys().toArray(String.class); logger.trace("performing state recovery from {}", Arrays.toString(nodesIds)); TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet(); logger.warn("failed to fetch state from node", failedNodeException); if (electedGlobalState == null) { electedGlobalState = nodeState.metaData(); } else if (nodeState.metaData().version() > electedGlobalState.version()) { electedGlobalState = nodeState.metaData(); for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) { indices.addTo(cursor.value.getIndex(), 1); MetaData.Builder metaDataBuilder = MetaData.builder(electedGlobalState).removeAllIndices(); continue; IndexMetaData indexMetaData = nodeState.metaData().index(index); if (indexMetaData == null) { continue; } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) { electedIndexMetaData = indexMetaData; logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
private Resolved resolveIndexPatterns(final String... requestedPatterns) { if(log.isTraceEnabled()) { log.trace("resolve requestedPatterns: "+Arrays.toString(requestedPatterns)); final SortedMap<String, AliasOrIndex> lookup = state.metaData().getAliasAndIndexLookup(); final Set<String> aliases = lookup.entrySet().stream().filter(e->e.getValue().isAlias()).map(e->e.getKey()).collect(Collectors.toSet()); try { _indices = new ArrayList<>(Arrays.asList(resolver.concreteIndexNames(state, IndicesOptions.fromOptions(false, true, true, false), requestedPatterns))); if (log.isDebugEnabled()) { log.debug("Resolved pattern {} to {}", requestedPatterns, _indices); Set<String> doubleIndices = lookup.get(al).getIndices().stream().map(a->a.getIndex().getName()).collect(Collectors.toSet()); _indices.removeAll(doubleIndices);
Void insertFirstMetaRow(final MetaData metadata, final String metaDataString) { try { logger.info(insertMetadataQuery); process(ConsistencyLevel.LOCAL_ONE, ClientState.forInternalCalls(), insertMetadataQuery, DatabaseDescriptor.getClusterName(), UUID.fromString(StorageService.instance.getLocalHostId()), metadata.version(), metaDataString); } catch (Exception e) { logger.error((Supplier<?>) () -> new ParameterizedMessage("Failed insert first row into table {}.{}", elasticAdminKeyspaceName, ELASTIC_ADMIN_METADATA_TABLE), e); throw e; } return null; }
final Set<String> excludeIndexPathIds = new HashSet<>(metaData.indices().size() + danglingIndices.size()); for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) { excludeIndexPathIds.add(cursor.value.getIndex().getUUID()); final List<IndexMetaData> indexMetaDataList = metaStateService.loadIndicesStates(excludeIndexPathIds::contains); Map<Index, IndexMetaData> newIndices = new HashMap<>(indexMetaDataList.size()); final IndexGraveyard graveyard = metaData.indexGraveyard(); for (IndexMetaData indexMetaData : indexMetaDataList) { if (metaData.hasIndex(indexMetaData.getIndex().getName())) { logger.warn("[{}] can not be imported as a dangling index, as index with same name already exists in cluster metadata", indexMetaData.getIndex()); } else if (graveyard.containsIndex(indexMetaData.getIndex())) { logger.warn("[{}] can not be imported as a dangling index, as an index with the same name and UUID exist in the " + "index tombstones. This situation is likely caused by copying over the data directory for an index " + "that was previously deleted.", indexMetaData.getIndex()); } else { logger.info("[{}] dangling index exists on local file system, but not in cluster metadata, " + "auto import to cluster state", indexMetaData.getIndex()); newIndices.put(indexMetaData.getIndex(), indexMetaData);
IndexMetaData indexMetaData = currentState.metaData().index(task.shardId.getIndex()); if (indexMetaData == null) { logger.debug("{} ignoring shard failed task [{}] (unknown index {})", task.shardId, task, task.shardId.getIndex()); batchResultBuilder.success(task); } else { long currentPrimaryTerm = indexMetaData.primaryTerm(task.shardId.id()); if (currentPrimaryTerm != task.primaryTerm) { assert currentPrimaryTerm > task.primaryTerm : "received a primary term with a higher term than in the " + "current cluster state (received [" + task.primaryTerm + "] but current is [" + currentPrimaryTerm + "])"; logger.debug("{} failing shard failed task [{}] (primary term {} does not match current term {})", task.shardId, task, task.primaryTerm, indexMetaData.primaryTerm(task.shardId.id())); batchResultBuilder.failure(task, new NoLongerPrimaryShardException( task.shardId, ShardRouting matched = currentState.getRoutingTable().getByAllocationId(task.shardId, task.allocationId); if (matched == null) { Set<String> inSyncAllocationIds = indexMetaData.inSyncAllocationIds(task.shardId.id()); logger.debug("{} marking shard {} as stale (shard failed task: [{}])", task.shardId, task.allocationId, task); tasksToBeApplied.add(task);
/** * Cleans dangling indices if they are already allocated on the provided meta data. */ void cleanupAllocatedDangledIndices(MetaData metaData) { for (Index index : danglingIndices.keySet()) { final IndexMetaData indexMetaData = metaData.index(index); if (indexMetaData != null && indexMetaData.getIndex().getName().equals(index.getName())) { if (indexMetaData.getIndex().getUUID().equals(index.getUUID()) == false) { logger.warn("[{}] can not be imported as a dangling index, as there is already another index " + "with the same name but a different uuid. local index will be ignored (but not deleted)", index); } else { logger.debug("[{}] no longer dangling (created), removing from dangling list", index); } danglingIndices.remove(index); } } }
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName()); logger.debug("{} deleted shard reason [{}]", shardId, reason); if (clusterState.nodes().getLocalNode().isMasterNode() == false && canDeleteIndexContents(shardId.getIndex(), indexSettings)) { if (nodeEnv.findAllShardIds(shardId.getIndex()).isEmpty()) { try { logger.trace("[{}] still has shard stores, leaving as is", shardId.getIndex());
private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException { logger.trace("listing store meta data for {}", shardId); long startTimeNS = System.nanoTime(); boolean exists = false; try { IndexService indexService = indicesService.indexService(shardId.getIndex()); if (indexService != null) { IndexShard indexShard = indexService.getShardOrNull(shardId.id()); if (indexShard != null) { exists = true; IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex()); if (metaData == null) { logger.trace("{} node doesn't have meta data for the requests index, responding with empty", shardId); return new StoreFilesMetaData(shardId, Store.MetadataSnapshot.EMPTY); TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS); if (exists) { logger.debug("{} loaded store meta data (took [{}])", shardId, took); } else { logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took);
public ClusterState closeIndices(ClusterState currentState, final Index[] indices, String indicesAsString) { Set<IndexMetaData> indicesToClose = new HashSet<>(); for (Index index : indices) { final IndexMetaData indexMetaData = currentState.metaData().getIndexSafe(index); if (indexMetaData.getState() != IndexMetaData.State.CLOSE) { indicesToClose.add(indexMetaData); logger.info("closing indices [{}]", indicesAsString); MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData()); ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder() .blocks(currentState.blocks()); for (IndexMetaData openIndexMetadata : indicesToClose) { final String indexName = openIndexMetadata.getIndex().getName(); mdBuilder.put(IndexMetaData.builder(openIndexMetadata).state(IndexMetaData.State.CLOSE)); blocksBuilder.addIndexBlock(indexName, INDEX_CLOSED_BLOCK); RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable()); for (IndexMetaData index : indicesToClose) { rtBuilder.remove(index.getIndex().getName());
final SortedMap<String, AliasOrIndex> lookup = clusterService.state().metaData().getAliasAndIndexLookup(); for(final String indexOrAlias: lookup.keySet()) { final String tenant = tenantNameForIndex(indexOrAlias); log.error(e1.toString(),e1);
AutoExpandReplicas.getAutoExpandReplicaChanges(clusterState.metaData(), clusterState.nodes()); if (autoExpandReplicaChanges.isEmpty()) { return clusterState; } else { final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); final MetaData.Builder metaDataBuilder = MetaData.builder(clusterState.metaData()); for (Map.Entry<Integer, List<String>> entry : autoExpandReplicaChanges.entrySet()) { final int numberOfReplicas = entry.getKey(); final IndexMetaData indexMetaData = metaDataBuilder.get(index); final IndexMetaData.Builder indexMetaDataBuilder = new IndexMetaData.Builder(indexMetaData).settingsVersion(1 + indexMetaData.getSettingsVersion()); metaDataBuilder.put(indexMetaDataBuilder); logger.info("updating number_of_replicas to [{}] for indices {}", numberOfReplicas, indices); final ClusterState fixedState = ClusterState.builder(clusterState).routingTable(routingTableBuilder.build())
@Override public ClusterState execute(ClusterState currentState) { Set<String> templateNames = new HashSet<>(); for (ObjectCursor<String> cursor : currentState.metaData().templates().keys()) { String templateName = cursor.value; if (Regex.simpleMatch(request.name, templateName)) { templateNames.add(templateName); } } if (templateNames.isEmpty()) { // if its a match all pattern, and no templates are found (we have none), don't // fail with index missing... if (Regex.isMatchAllPattern(request.name)) { return currentState; } throw new IndexTemplateMissingException(request.name); } MetaData.Builder metaData = MetaData.builder(currentState.metaData()); for (String templateName : templateNames) { logger.info("removing template [{}]", templateName); metaData.removeTemplate(templateName); } return ClusterState.builder(currentState).metaData(metaData).build(); }
@Override public ClusterState execute(ClusterState currentState) throws Exception { if (request.create && currentState.metaData().templates().containsKey(request.name)) { throw new IllegalArgumentException("index_template [" + request.name + "] already exists"); } validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry); for (Alias alias : request.aliases) { AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter()) .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build(); templateBuilder.putAlias(aliasMetaData); } IndexTemplateMetaData template = templateBuilder.build(); MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template); logger.info("adding template [{}] for index patterns {}", request.name, request.indexPatterns); return ClusterState.builder(currentState).metaData(builder).build(); }
final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); if (localRoutingNode != null) { // null e.g. if we are not a data node for (ShardRouting shardRouting : localRoutingNode) { indicesWithShards.add(shardRouting.index()); final IndexMetaData indexMetaData = state.metaData().index(index); assert indexMetaData != null || event.isNewCluster() : "index " + index + " does not exist in the cluster state, it should either " + "have been deleted or the cluster must be new"; final AllocatedIndices.IndexRemovalReason reason = indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED; logger.debug("{} removing index, [{}]", index, reason); indicesService.removeIndex(index, reason, "removing index (no shards allocated)");
MetaData oldMetaData = clusterState.metaData(); RoutingTable oldRoutingTable = clusterState.routingTable(); MetaData.Builder metaDataBuilder = null; Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) { final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey()); IndexMetaData.Builder indexMetaDataBuilder = null; int shardNumber = shardEntry.getKey().getId(); Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber); Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet()); assert idsToRemove.stream().allMatch(id -> oldRoutingTable.getByAllocationId(shardEntry.getKey(), id) == null) : indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData); logger.warn("{} marking unavailable shards as stale: {}", shardEntry.getKey(), idsToRemove); metaDataBuilder = MetaData.builder(oldMetaData); return ClusterState.builder(clusterState).metaData(metaDataBuilder).build(); } else { return clusterState;
/** * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index * but does not deal with in-memory structures. For those call {@link #removeIndex(Index, IndexRemovalReason, String)} */ @Override public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) { if (nodeEnv.hasNodeFile()) { String indexName = metaData.getIndex().getName(); try { if (clusterState.metaData().hasIndex(indexName)) { final IndexMetaData index = clusterState.metaData().index(indexName); throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " + "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]"); } deleteIndexStore(reason, metaData, clusterState); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e); } } }