final IndexMetaData indexMetaData = clusterService.state().metaData().getIndices().get(requestAliasOrIndex); log.debug("{} does not exist in cluster metadata", requestAliasOrIndex); continue; final ImmutableOpenMap<String, AliasMetaData> aliases = indexMetaData.getAliases(); if(aliases != null && aliases.size() > 0) { if(log.isDebugEnabled()) { log.debug("Aliases for {}: {}", requestAliasOrIndex, aliases); final Iterator<String> it = aliases.keysIt(); while(it.hasNext()) { final String alias = it.next(); final AliasMetaData aliasMetaData = aliases.get(alias);
private String[] getIndexesFromEs(final String aliasName){ final AdminClient adminClient = this.provider.getClient().admin(); //remove write alias, can only have one ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases(); return aliasMap.keys().toArray( String.class ); }
logger.trace("dropRecreateMapping index[{}] - type[{}]", index, type); client.admin().indices().prepareRefresh(index).get(); ImmutableOpenMap<String, MappingMetaData> mappings = client.admin().cluster().prepareState().get().getState().getMetaData() .index(index).mappings(); logger.trace("mappings contains type {}: {}", type, mappings.containsKey(type)); if (mappings.containsKey(type)) { MappingMetaData mapping = mappings.get(type); if (client.admin().indices().prepareDeleteMapping(index).setType(type).get().isAcknowledged()) { PutMappingResponse pmr = client.admin().indices().preparePutMapping(index).setType(type)
final IndexMetaData indexMetaData = currentState.getMetaData().getIndexSafe(index); for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) { String parentType = newMapper.parentFieldMapper().type(); if (parentType.equals(mapping.value.type()) && throw new InvalidTypeNameException("Document mapping type name can't start with '_', found: [" + mappingType + "]"); MetaData.Builder builder = MetaData.builder(metaData); boolean updated = false; for (IndexMetaData indexMetaData : updateList) { final Index index = indexMetaData.getIndex(); final MapperService mapperService = indexMapperServices.get(index); CompressedXContent existingSource = null; updatedMapping = true; if (logger.isDebugEnabled()) { logger.debug("{} update_mapping [{}] with source [{}]", index, mergedMapper.type(), updatedSource); } else if (logger.isInfoEnabled()) { logger.info("{} update_mapping [{}]", index, mergedMapper.type()); IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); return ClusterState.builder(currentState).metaData(builder).build(); } else { return currentState;
public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException { String[] nodesIds = clusterService.state().nodes().getMasterNodes().keys().toArray(String.class); logger.trace("performing state recovery from {}", Arrays.toString(nodesIds)); TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet(); logger.warn("failed to fetch state from node", failedNodeException); if (electedGlobalState == null) { electedGlobalState = nodeState.metaData(); } else if (nodeState.metaData().version() > electedGlobalState.version()) { electedGlobalState = nodeState.metaData(); for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) { indices.addTo(cursor.value.getIndex(), 1); } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) { electedIndexMetaData = indexMetaData; logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
final Set<String> excludeIndexPathIds = new HashSet<>(metaData.indices().size() + danglingIndices.size()); for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) { excludeIndexPathIds.add(cursor.value.getIndex().getUUID()); final List<IndexMetaData> indexMetaDataList = metaStateService.loadIndicesStates(excludeIndexPathIds::contains); Map<Index, IndexMetaData> newIndices = new HashMap<>(indexMetaDataList.size()); final IndexGraveyard graveyard = metaData.indexGraveyard(); for (IndexMetaData indexMetaData : indexMetaDataList) { if (metaData.hasIndex(indexMetaData.getIndex().getName())) { logger.warn("[{}] can not be imported as a dangling index, as index with same name already exists in cluster metadata", indexMetaData.getIndex()); } else if (graveyard.containsIndex(indexMetaData.getIndex())) { logger.warn("[{}] can not be imported as a dangling index, as an index with the same name and UUID exist in the " + "index tombstones. This situation is likely caused by copying over the data directory for an index " + "that was previously deleted.", indexMetaData.getIndex()); } else { logger.info("[{}] dangling index exists on local file system, but not in cluster metadata, " + "auto import to cluster state", indexMetaData.getIndex()); newIndices.put(indexMetaData.getIndex(), indexMetaData);
if (this.isMaster && dataNodeAdded && event.state().getNodes().getDataNodes().size() > 1) { if (logger.isDebugEnabled()) { logger.debug("data node was added, retrieving new cluster info"); for (DiscoveryNode removedNode : event.nodesDelta().removedNodes()) { if (removedNode.isDataNode()) { if (logger.isTraceEnabled()) { logger.trace("Removing node from cluster info: {}", removedNode.getId()); if (leastAvailableSpaceUsages.containsKey(removedNode.getId())) { ImmutableOpenMap.Builder<String, DiskUsage> newMaxUsages = ImmutableOpenMap.builder(leastAvailableSpaceUsages); newMaxUsages.remove(removedNode.getId()); leastAvailableSpaceUsages = newMaxUsages.build(); if (mostAvailableSpaceUsages.containsKey(removedNode.getId())) { ImmutableOpenMap.Builder<String, DiskUsage> newMinUsages = ImmutableOpenMap.builder(mostAvailableSpaceUsages); newMinUsages.remove(removedNode.getId()); mostAvailableSpaceUsages = newMinUsages.build();
@Override public void onMaster() { this.isMaster = true; if (logger.isTraceEnabled()) { logger.trace("I have been elected master, scheduling a ClusterInfoUpdateJob"); } try { // Submit a job that will start after DEFAULT_STARTING_INTERVAL, and reschedule itself after running threadPool.schedule(updateFrequency, executorName(), new SubmitReschedulingClusterInfoUpdatedJob()); if (clusterService.state().getNodes().getDataNodes().size() > 1) { // Submit an info update job to be run immediately threadPool.executor(executorName()).execute(() -> maybeRefresh()); } } catch (EsRejectedExecutionException ex) { if (logger.isDebugEnabled()) { logger.debug("Couldn't schedule cluster info update task - node might be shutting down", ex); } } }
private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation, ImmutableOpenMap<String, DiskUsage> usages, boolean subtractLeavingShards) { DiskUsage usage = usages.get(node.nodeId()); if (usage == null) { // If there is no usage, and we have other nodes in the cluster, // use the average usage for all nodes as the usage for this node usage = averageUsage(node, usages); if (logger.isDebugEnabled()) { logger.debug("unable to determine disk usage for {}, defaulting to average across nodes [{} total] [{} free] [{}% free]", node.nodeId(), usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeDiskAsPercentage()); } } if (diskThresholdSettings.includeRelocations()) { long relocatingShardsSize = sizeOfRelocatingShards(node, allocation, subtractLeavingShards, usage.getPath()); DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().getName(), usage.getPath(), usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize); if (logger.isTraceEnabled()) { logger.trace("usage without relocations: {}", usage); logger.trace("usage with relocations: [{} bytes] {}", relocatingShardsSize, usageIncludingRelocations); } usage = usageIncludingRelocations; } return usage; }
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName()); logger.debug("{} deleted shard reason [{}]", shardId, reason); if (clusterState.nodes().getLocalNode().isMasterNode() == false && canDeleteIndexContents(shardId.getIndex(), indexSettings)) { if (nodeEnv.findAllShardIds(shardId.getIndex()).isEmpty()) { logger.trace("[{}] still has shard stores, leaving as is", shardId.getIndex());
@Override protected void doRun() { final Snapshot snapshot = entry.snapshot(); final Repository repository = repositoriesService.repository(snapshot.getRepository()); logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure); ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>(); for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) { ShardId shardId = shardStatus.key; ShardSnapshotStatus status = shardStatus.value; if (status.state().failed()) { shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason())); } } SnapshotInfo snapshotInfo = repository.finalizeSnapshot( snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures), entry.getRepositoryStateId(), entry.includeGlobalState()); removeSnapshotFromClusterState(snapshot, snapshotInfo, null); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); }
@Override public ClusterState execute(ClusterState currentState) { Set<String> templateNames = new HashSet<>(); for (ObjectCursor<String> cursor : currentState.metaData().templates().keys()) { String templateName = cursor.value; if (Regex.simpleMatch(request.name, templateName)) { templateNames.add(templateName); } } if (templateNames.isEmpty()) { // if its a match all pattern, and no templates are found (we have none), don't // fail with index missing... if (Regex.isMatchAllPattern(request.name)) { return currentState; } throw new IndexTemplateMissingException(request.name); } MetaData.Builder metaData = MetaData.builder(currentState.metaData()); for (String templateName : templateNames) { logger.info("removing template [{}]", templateName); metaData.removeTemplate(templateName); } return ClusterState.builder(currentState).metaData(metaData).build(); }
@Override public ClusterState execute(ClusterState currentState) throws Exception { if (request.create && currentState.metaData().templates().containsKey(request.name)) { throw new IllegalArgumentException("index_template [" + request.name + "] already exists"); } validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry); for (Alias alias : request.aliases) { AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter()) .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build(); templateBuilder.putAlias(aliasMetaData); } IndexTemplateMetaData template = templateBuilder.build(); MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template); logger.info("adding template [{}] for index patterns {}", request.name, request.indexPatterns); return ClusterState.builder(currentState).metaData(builder).build(); }
SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null) { return; final String localNodeId = event.state().nodes().getLocalNodeId(); final DiscoveryNode masterNode = event.state().nodes().getMasterNode(); for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) { if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) { ShardId shardId = localShard.getKey(); IndexShardSnapshotStatus localShardStatus = localShard.getValue(); ShardSnapshotStatus masterShard = masterShards.get(shardId); if (masterShard != null && masterShard.state().completed() == false) { final IndexShardSnapshotStatus.Copy indexShardSnapshotStatus = localShard.getValue().asCopy(); logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, " + "updating status on the master", snapshot.snapshot(), shardId); notifySuccessfulSnapshotShard(snapshot.snapshot(), shardId, localNodeId, masterNode); logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, " + "updating status on master", snapshot.snapshot(), shardId); final String failure = indexShardSnapshotStatus.getFailure();
nodesFD.stop(); DiscoveryNodes nodes = clusterState().nodes(); if (sendLeaveRequest) { if (nodes.getMasterNode() == null) { } else if (!nodes.isLocalNodeElectedMaster()) { try { membership.sendLeaveRequestBlocking(nodes.getMasterNode(), nodes.getLocalNode(), TimeValue.timeValueSeconds(1)); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request to master [{}]", nodes.getMasterNode()), e); DiscoveryNode[] possibleMasters = electMaster.nextPossibleMasters(nodes.getNodes().values(), 5); for (DiscoveryNode possibleMaster : possibleMasters) { if (nodes.getLocalNode().equals(possibleMaster)) { membership.sendLeaveRequest(nodes.getLocalNode(), possibleMaster); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request from master [{}] to possible master [{}]", nodes.getMasterNode(), possibleMaster), e);
public void logMinimumMasterNodesWarningIfNecessary(ClusterState oldState, ClusterState newState) { // check if min_master_nodes setting is too low and log warning if (hasTooManyMasterNodes(oldState.nodes()) == false && hasTooManyMasterNodes(newState.nodes())) { logger.warn("value for setting \"{}\" is too low. This can result in data loss! Please set it to at least a quorum of master-" + "eligible nodes (current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])", ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNodes(), newState.getNodes().getMasterNodes().size()); } }
public void verify(String repository, boolean readOnly, String verificationToken, final ActionListener<VerifyResponse> listener) { final DiscoveryNodes discoNodes = clusterService.state().nodes(); final DiscoveryNode localNode = discoNodes.getLocalNode(); final ObjectContainer<DiscoveryNode> masterAndDataNodes = discoNodes.getMasterAndDataNodes().values(); final List<DiscoveryNode> nodes = new ArrayList<>(); for (ObjectCursor<DiscoveryNode> cursor : masterAndDataNodes) { doVerify(repository, verificationToken, localNode); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); errors.add(new VerificationFailure(node.getId(), e));
private Decision earlyTerminate(RoutingAllocation allocation, ImmutableOpenMap<String, DiskUsage> usages) { // Always allow allocation if the decider is disabled if (diskThresholdSettings.isEnabled() == false) { return allocation.decision(Decision.YES, NAME, "the disk threshold decider is disabled"); } // Allow allocation regardless if only a single data node is available if (allocation.nodes().getDataNodes().size() <= 1) { if (logger.isTraceEnabled()) { logger.trace("only a single data node is present, allowing allocation"); } return allocation.decision(Decision.YES, NAME, "there is only a single data node present"); } // Fail open there is no info available final ClusterInfo clusterInfo = allocation.clusterInfo(); if (clusterInfo == null) { if (logger.isTraceEnabled()) { logger.trace("cluster info unavailable for disk threshold decider, allowing allocation."); } return allocation.decision(Decision.YES, NAME, "the cluster info is unavailable"); } // Fail open if there are no disk usages available if (usages.isEmpty()) { if (logger.isTraceEnabled()) { logger.trace("unable to determine disk usages for disk-aware allocation, allowing allocation"); } return allocation.decision(Decision.YES, NAME, "disk usages are unavailable"); } return null; }