public Boolean hasNode(DiscoveryNode node) { if(nodes == null) { if(log.isDebugEnabled()) { log.debug("Cluster Info Holder not initialized yet for 'nodes'"); } return null; } return nodes.nodeExists(node)?Boolean.TRUE:Boolean.FALSE; }
/** * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not. */ private static DiscoveryNode findSourceNodeForPeerRecovery(Logger logger, RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) { DiscoveryNode sourceNode = null; if (!shardRouting.primary()) { ShardRouting primary = routingTable.shardRoutingTable(shardRouting.shardId()).primaryShard(); // only recover from started primary, if we can't find one, we will do it next round if (primary.active()) { sourceNode = nodes.get(primary.currentNodeId()); if (sourceNode == null) { logger.trace("can't find replica source node because primary shard {} is assigned to an unknown node.", primary); } } else { logger.trace("can't find replica source node because primary shard {} is not active.", primary); } } else if (shardRouting.relocatingNodeId() != null) { sourceNode = nodes.get(shardRouting.relocatingNodeId()); if (sourceNode == null) { logger.trace("can't find relocation source node for shard {} because it is assigned to an unknown node [{}].", shardRouting.shardId(), shardRouting.relocatingNodeId()); } } else { throw new IllegalStateException("trying to find source node for peer recovery when routing state means no peer recovery: " + shardRouting); } return sourceNode; }
/** * In the case we follow an elected master the new cluster state needs to have the same elected master * This method checks for this and throws an exception if needed */ public static void validateStateIsFromCurrentMaster(Logger logger, DiscoveryNodes currentNodes, ClusterState newClusterState) { if (currentNodes.getMasterNodeId() == null) { return; } if (!currentNodes.getMasterNodeId().equals(newClusterState.nodes().getMasterNodeId())) { logger.warn("received a cluster state from a different master than the current one, rejecting (received {}, current {})", newClusterState.nodes().getMasterNode(), currentNodes.getMasterNode()); throw new IllegalStateException("cluster state from a different master than the current one, rejecting (received " + newClusterState.nodes().getMasterNode() + ", current " + currentNodes.getMasterNode() + ")"); } }
private PingResponse createPingResponse(ClusterState clusterState) { DiscoveryNodes discoNodes = clusterState.nodes(); return new PingResponse(discoNodes.getLocalNode(), discoNodes.getMasterNode(), clusterState); }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("nodes: \n"); for (DiscoveryNode node : this) { sb.append(" ").append(node); if (node == getLocalNode()) { sb.append(", local"); } if (node == getMasterNode()) { sb.append(", master"); } sb.append("\n"); } return sb.toString(); }
final ConcurrentMap<String, PreSyncedFlushResponse> presyncResponses = ConcurrentCollections.newConcurrentMap(); for (final ShardRouting shard : shards) { logger.trace("{} sending pre-synced flush request to {}", shardId, shard); final DiscoveryNode node = state.nodes().get(shard.currentNodeId()); if (node == null) { logger.trace("{} shard routing {} refers to an unknown node. skipping.", shardId, shard); if (countDown.countDown()) { listener.onResponse(presyncResponses); transportService.sendRequest(node, PRE_SYNCED_FLUSH_ACTION_NAME, new PreShardSyncedFlushRequest(shard.shardId()), new TransportResponseHandler<PreSyncedFlushResponse>() { @Override
assert newClusterState.nodes().getMasterNode() != null : "received a cluster state without a master"; assert !newClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock()) : "received a cluster state with a master block"; if (currentState.nodes().isLocalNodeElectedMaster() && newClusterState.nodes().isLocalNodeElectedMaster() == false) { handleAnotherMaster(currentState, newClusterState.nodes().getMasterNode(), newClusterState.version(), "via a new cluster state"); return false; newClusterState.version(), newClusterState.stateUUID(), newClusterState.nodes().getMasterNodeId() ); throw new IllegalStateException(message); } catch (Exception inner) { inner.addSuppressed(e); logger.error(() -> new ParameterizedMessage("unexpected exception while failing [{}]", reason), inner); if (currentState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock())) { logger.debug("got first state from fresh master [{}]", newClusterState.nodes().getMasterNodeId()); if (newClusterState.nodes().isLocalNodeElectedMaster()) { if (masterFD.masterNode() == null || !masterFD.masterNode().equals(newClusterState.nodes().getMasterNode())) { masterFD.restart(newClusterState.nodes().getMasterNode(), "new cluster state received and we are monitoring the wrong master [" + masterFD.masterNode() + "]");
@Override public void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener) { ClusterState newState = clusterChangedEvent.state(); assert newState.getNodes().isLocalNodeElectedMaster() : "Shouldn't publish state when not master " + clusterChangedEvent.source(); } catch (FailedToCommitClusterStateException t) { logger.debug("failed to publish cluster state version [{}] (not enough nodes acknowledged, min master nodes [{}])", newState.version(), electMaster.minimumMasterNodes()); final DiscoveryNode localNode = newState.getNodes().getLocalNode(); final CountDownLatch latch = new CountDownLatch(1); final AtomicBoolean processedOrFailed = new AtomicBoolean(); boolean sentToApplier = processNextCommittedClusterState("master " + newState.nodes().getMasterNode() + " committed version [" + newState.version() + "] source [" + clusterChangedEvent.source() + "]"); if (sentToApplier == false && processedOrFailed.get() == false) { assert false : "cluster state published locally neither processed nor failed: " + newState; logger.warn("cluster state with version [{}] that is published locally has neither been processed nor failed", newState.version()); return; latch.await(); } catch (InterruptedException e) { logger.debug(() -> new ParameterizedMessage( "interrupted while applying cluster state locally [{}]", clusterChangedEvent.source()), e); Thread.currentThread().interrupt();
final ClusterTasksResult.Builder<DiscoveryNode> results = ClusterTasksResult.builder(); if (joiningNodes.size() == 1 && joiningNodes.get(0).equals(FINISH_ELECTION_TASK)) { return results.successes(joiningNodes).build(currentState); } else if (currentNodes.getMasterNode() == null && joiningNodes.contains(BECOME_MASTER_TASK)) { assert joiningNodes.contains(FINISH_ELECTION_TASK) : "becoming a master but election is not finished " + joiningNodes; } else if (currentNodes.isLocalNodeElectedMaster() == false) { logger.trace("processing node joins, but we are not the master. current master: {}", currentNodes.getMasterNode()); throw new NotMasterException("Node [" + currentNodes.getLocalNode() + "] not master for join request"); } else { newState = ClusterState.builder(currentState); DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(newState.nodes()); Version minClusterNodeVersion = newState.nodes().getMinNodeVersion(); Version maxClusterNodeVersion = newState.nodes().getMaxNodeVersion(); final boolean enforceMajorVersion = currentState.getBlocks().hasGlobalBlock(STATE_NOT_RECOVERED_BLOCK) == false; if (node.equals(BECOME_MASTER_TASK) || node.equals(FINISH_ELECTION_TASK)) { } else if (currentNodes.nodeExists(node)) { logger.debug("received a join request for an existing node [{}]", node); } else { try {
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName()); logger.debug("{} deleted shard reason [{}]", shardId, reason); if (clusterState.nodes().getLocalNode().isMasterNode() == false && canDeleteIndexContents(shardId.getIndex(), indexSettings)) { if (nodeEnv.findAllShardIds(shardId.getIndex()).isEmpty()) { logger.trace("[{}] still has shard stores, leaving as is", shardId.getIndex());
private void createIndices(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { return; if (failedShardsCache.containsKey(shardRouting.shardId()) == false) { final Index index = shardRouting.index(); if (indicesService.indexService(index) == null) { indicesToCreate.computeIfAbsent(index, k -> new ArrayList<>()).add(shardRouting); final IndexMetaData indexMetaData = state.metaData().index(index); logger.debug("[{}] creating index", index); indexService = indicesService.createIndex(indexMetaData, buildInIndexListener); if (indexService.updateMapping(null, indexMetaData) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(), indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId()) );
SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> survivors = new HashMap<>(); final String localNodeId = event.state().nodes().getLocalNodeId(); final DiscoveryNode masterNode = event.state().nodes().getMasterNode(); final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>(); if (snapshotsInProgress != null) { logger.trace("[{}] - Adding shard to the queue", shard.key); startedShards.put(shard.key, IndexShardSnapshotStatus.newInitializing()); final Stage stage = lastSnapshotStatus.getStage(); if (stage == Stage.FINALIZE) { logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, " + "letting it finish", entry.snapshot(), shard.key); logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, " + "updating status on the master", entry.snapshot(), shard.key); notifySuccessfulSnapshotShard(entry.snapshot(), shard.key, localNodeId, masterNode); logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to snapshot shard", shardId, snapshot), e); failure.set(e);
private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting, ClusterState state) { assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting; DiscoveryNode sourceNode = null; if (shardRouting.recoverySource().getType() == Type.PEER) { sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting); if (sourceNode == null) { logger.trace("ignoring initializing shard {} - no source node can be found.", shardRouting.shardId()); return; } } try { logger.debug("{} creating shard", shardRouting.shardId()); RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode); indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting), repositoriesService, failedShardHandler, globalCheckpointSyncer); } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed to create shard", e, state); } }
DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode(); if (localNode == null) { logger.warn("couldn't store response {}, the node didn't join the cluster yet", response); listener.onResponse(response); return; taskResult = task.result(localNode, response); } catch (IOException ex) { logger.warn(() -> new ParameterizedMessage("couldn't store response {}", response), ex); listener.onFailure(ex); return;
private void handleMasterGone(final DiscoveryNode masterNode, final Throwable cause, final String reason) { if (lifecycleState() != Lifecycle.State.STARTED) { // not started, ignore a master failure return; } if (localNodeMaster()) { // we might get this on both a master telling us shutting down, and then the disconnect failure return; } logger.info(() -> new ParameterizedMessage("master_left [{}], reason [{}]", masterNode, reason), cause); synchronized (stateMutex) { if (localNodeMaster() == false && masterNode.equals(committedState.get().nodes().getMasterNode())) { // flush any pending cluster states from old master, so it will not be set as master again pendingStatesQueue.failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason)); rejoin("master left (reason = " + reason + ")"); } } }
private ClusterState.Builder becomeMasterAndTrimConflictingNodes(ClusterState currentState, List<DiscoveryNode> joiningNodes) { assert currentState.nodes().getMasterNodeId() == null : currentState; DiscoveryNodes currentNodes = currentState.nodes(); DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes); nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId()); for (final DiscoveryNode joiningNode : joiningNodes) { final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId()); if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameId, joiningNode); nodesBuilder.remove(nodeWithSameId.getId()); } final DiscoveryNode nodeWithSameAddress = currentNodes.findByAddress(joiningNode.getAddress()); if (nodeWithSameAddress != null && nodeWithSameAddress.equals(joiningNode) == false) { logger.debug("removing existing node [{}], which conflicts with incoming join from [{}]", nodeWithSameAddress, joiningNode); nodesBuilder.remove(nodeWithSameAddress.getId()); } } // now trim any left over dead nodes - either left there when the previous master stepped down // or removed by us above ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).blocks(ClusterBlocks.builder() .blocks(currentState.blocks()) .removeGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)).build(); return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false, "removed dead nodes on election")); }
nodesFD.stop(); DiscoveryNodes nodes = clusterState().nodes(); if (sendLeaveRequest) { if (nodes.getMasterNode() == null) { } else if (!nodes.isLocalNodeElectedMaster()) { try { membership.sendLeaveRequestBlocking(nodes.getMasterNode(), nodes.getLocalNode(), TimeValue.timeValueSeconds(1)); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request to master [{}]", nodes.getMasterNode()), e); DiscoveryNode[] possibleMasters = electMaster.nextPossibleMasters(nodes.getNodes().values(), 5); for (DiscoveryNode possibleMaster : possibleMasters) { if (nodes.getLocalNode().equals(possibleMaster)) { continue; membership.sendLeaveRequest(nodes.getLocalNode(), possibleMaster); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request from master [{}] to possible master [{}]", nodes.getMasterNode(), possibleMaster), e);
public void verify(String repository, boolean readOnly, String verificationToken, final ActionListener<VerifyResponse> listener) { final DiscoveryNodes discoNodes = clusterService.state().nodes(); final DiscoveryNode localNode = discoNodes.getLocalNode(); final ObjectContainer<DiscoveryNode> masterAndDataNodes = discoNodes.getMasterAndDataNodes().values(); final List<DiscoveryNode> nodes = new ArrayList<>(); for (ObjectCursor<DiscoveryNode> cursor : masterAndDataNodes) { DiscoveryNode node = cursor.value; if (readOnly && node.getVersion().before(Version.V_6_6_0)) { continue; final AtomicInteger counter = new AtomicInteger(nodes.size()); for (final DiscoveryNode node : nodes) { if (node.equals(localNode)) { try { doVerify(repository, verificationToken, localNode); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to verify repository", repository), e); errors.add(new VerificationFailure(node.getId(), e));
SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null) { return; final String localNodeId = event.state().nodes().getLocalNodeId(); final DiscoveryNode masterNode = event.state().nodes().getMasterNode(); for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) { if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) { ShardId shardId = localShard.getKey(); IndexShardSnapshotStatus localShardStatus = localShard.getValue(); ShardSnapshotStatus masterShard = masterShards.get(shardId); if (masterShard != null && masterShard.state().completed() == false) { final IndexShardSnapshotStatus.Copy indexShardSnapshotStatus = localShard.getValue().asCopy(); logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, " + "updating status on the master", snapshot.snapshot(), shardId); notifySuccessfulSnapshotShard(snapshot.snapshot(), shardId, localNodeId, masterNode); logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, " + "updating status on master", snapshot.snapshot(), shardId); final String failure = indexShardSnapshotStatus.getFailure();