/** * In the case we follow an elected master the new cluster state needs to have the same elected master * This method checks for this and throws an exception if needed */ public static void validateStateIsFromCurrentMaster(Logger logger, DiscoveryNodes currentNodes, ClusterState newClusterState) { if (currentNodes.getMasterNodeId() == null) { return; } if (!currentNodes.getMasterNodeId().equals(newClusterState.nodes().getMasterNodeId())) { logger.warn("received a cluster state from a different master than the current one, rejecting (received {}, current {})", newClusterState.nodes().getMasterNode(), currentNodes.getMasterNode()); throw new IllegalStateException("cluster state from a different master than the current one, rejecting (received " + newClusterState.nodes().getMasterNode() + ", current " + currentNodes.getMasterNode() + ")"); } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("nodes: \n"); for (DiscoveryNode node : this) { sb.append(" ").append(node); if (node == getLocalNode()) { sb.append(", local"); } if (node == getMasterNode()) { sb.append(", master"); } sb.append("\n"); } return sb.toString(); }
/** * Returns the changes comparing this nodes to the provided nodes. */ public Delta delta(DiscoveryNodes other) { List<DiscoveryNode> removed = new ArrayList<>(); List<DiscoveryNode> added = new ArrayList<>(); for (DiscoveryNode node : other) { if (!this.nodeExists(node)) { removed.add(node); } } for (DiscoveryNode node : this) { if (!other.nodeExists(node)) { added.add(node); } } DiscoveryNode previousMasterNode = null; DiscoveryNode newMasterNode = null; if (masterNodeId != null) { if (other.masterNodeId == null || !other.masterNodeId.equals(masterNodeId)) { previousMasterNode = other.getMasterNode(); newMasterNode = getMasterNode(); } } return new Delta(previousMasterNode, newMasterNode, localNodeId, Collections.unmodifiableList(removed), Collections.unmodifiableList(added)); }
/** * does simple sanity check of the incoming cluster state. Throws an exception on rejections. */ static void validateIncomingState(Logger logger, ClusterState incomingState, ClusterState lastState) { final ClusterName incomingClusterName = incomingState.getClusterName(); if (!incomingClusterName.equals(lastState.getClusterName())) { logger.warn("received cluster state from [{}] which is also master but with a different cluster name [{}]", incomingState.nodes().getMasterNode(), incomingClusterName); throw new IllegalStateException("received state from a node that is not part of the cluster"); } if (lastState.nodes().getLocalNode().equals(incomingState.nodes().getLocalNode()) == false) { logger.warn("received a cluster state from [{}] and not part of the cluster, should not happen", incomingState.nodes().getMasterNode()); throw new IllegalStateException("received state with a local node that does not match the current local node"); } if (shouldIgnoreOrRejectNewClusterState(logger, lastState, incomingState)) { String message = String.format( Locale.ROOT, "rejecting cluster state version [%d] uuid [%s] received from [%s]", incomingState.version(), incomingState.stateUUID(), incomingState.nodes().getMasterNodeId() ); logger.warn(message); throw new IllegalStateException(message); } }
AckCountDownListener(AckedClusterStateTaskListener ackedTaskListener, long clusterStateVersion, DiscoveryNodes nodes, ThreadPool threadPool) { this.ackedTaskListener = ackedTaskListener; this.clusterStateVersion = clusterStateVersion; this.threadPool = threadPool; this.masterNode = nodes.getMasterNode(); int countDown = 0; for (DiscoveryNode node : nodes) { //we always wait for at least the master node if (node.equals(masterNode) || ackedTaskListener.mustAck(node)) { countDown++; } } logger.trace("expecting {} acknowledgements for cluster_state update (version: {})", countDown, clusterStateVersion); this.countDown = new CountDown(countDown + 1); // we also wait for onCommit to be called }
private PingResponse createPingResponse(ClusterState clusterState) { DiscoveryNodes discoNodes = clusterState.nodes(); return new PingResponse(discoNodes.getLocalNode(), discoNodes.getMasterNode(), clusterState); }
/** * builds a predicate that will accept a cluster state only if it was generated after the current has * (re-)joined the master */ public static Predicate<ClusterState> build(ClusterState currentState) { final long currentVersion = currentState.version(); final DiscoveryNode masterNode = currentState.nodes().getMasterNode(); final String currentMasterId = masterNode == null ? null : masterNode.getEphemeralId(); return newState -> { final DiscoveryNode newMaster = newState.nodes().getMasterNode(); final boolean accept; if (newMaster == null) { accept = false; } else if (newMaster.getEphemeralId().equals(currentMasterId) == false) { accept = true; } else { accept = newState.version() > currentVersion; } return accept; }; } }
/** * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master. * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node. * Resends shard failures for shards that are still marked as allocated to this node but previously failed. * * @param state new cluster state */ private void updateFailedShardsCache(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { failedShardsCache.clear(); return; } DiscoveryNode masterNode = state.nodes().getMasterNode(); // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) { ShardRouting failedShardRouting = iterator.next().getValue(); ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId()); if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) { iterator.remove(); } else { if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction? String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure"; logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message); shardStateAction.localShardFailed(matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER, state); } } } }
@Override public void onClusterStateCommitted(String stateUUID, ActionListener<Void> processedListener) { final ClusterState state = pendingStatesQueue.markAsCommitted(stateUUID, new PendingClusterStatesQueue.StateProcessedListener() { @Override public void onNewClusterStateProcessed() { processedListener.onResponse(null); } @Override public void onNewClusterStateFailed(Exception e) { processedListener.onFailure(e); } }); if (state != null) { synchronized (stateMutex) { processNextCommittedClusterState("master " + state.nodes().getMasterNode() + " committed version [" + state.version() + "]"); } } }
private void handleMasterGone(final DiscoveryNode masterNode, final Throwable cause, final String reason) { if (lifecycleState() != Lifecycle.State.STARTED) { // not started, ignore a master failure return; } if (localNodeMaster()) { // we might get this on both a master telling us shutting down, and then the disconnect failure return; } logger.info(() -> new ParameterizedMessage("master_left [{}], reason [{}]", masterNode, reason), cause); synchronized (stateMutex) { if (localNodeMaster() == false && masterNode.equals(committedState.get().nodes().getMasterNode())) { // flush any pending cluster states from old master, so it will not be set as master again pendingStatesQueue.failAllStatesAndClear(new ElasticsearchException("master left [{}]", reason)); rejoin("master left (reason = " + reason + ")"); } } }
public void allocateDangled(Collection<IndexMetaData> indices, final Listener listener) { ClusterState clusterState = clusterService.state(); DiscoveryNode masterNode = clusterState.nodes().getMasterNode(); if (masterNode == null) { listener.onFailure(new MasterNotDiscoveredException("no master to send allocate dangled request")); return; } AllocateDangledRequest request = new AllocateDangledRequest(clusterService.localNode(), indices.toArray(new IndexMetaData[indices.size()])); transportService.sendRequest(masterNode, ACTION_NAME, request, new TransportResponseHandler<AllocateDangledResponse>() { @Override public AllocateDangledResponse read(StreamInput in) throws IOException { final AllocateDangledResponse response = new AllocateDangledResponse(); response.readFrom(in); return response; } @Override public void handleResponse(AllocateDangledResponse response) { listener.onResponse(response); } @Override public void handleException(TransportException exp) { listener.onFailure(exp); } @Override public String executor() { return ThreadPool.Names.SAME; } }); }
if (nodes.getMasterNode() == null) { membership.sendLeaveRequestBlocking(nodes.getMasterNode(), nodes.getLocalNode(), TimeValue.timeValueSeconds(1)); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request to master [{}]", nodes.getMasterNode()), e); } catch (Exception e) { logger.debug(() -> new ParameterizedMessage("failed to send leave request from master [{}] to possible master [{}]", nodes.getMasterNode(), possibleMaster), e);
ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext()); DiscoveryNode masterNode = currentState.nodes().getMasterNode(); Predicate<ClusterState> changePredicate = MasterNodeChangePredicate.build(currentState); if (masterNode == null) {
private void handleLeaveRequest(final DiscoveryNode node) { if (lifecycleState() != Lifecycle.State.STARTED) { // not started, ignore a node failure return; } if (localNodeMaster()) { removeNode(node, "zen-disco-node-left", "left"); } else if (node.equals(clusterState().nodes().getMasterNode())) { handleMasterGone(node, null, "shut_down"); } }
+ "], version [" + state.version() + "]"); final DiscoveryNode currentMaster = state.nodes().getMasterNode(); assert currentMaster != null : "processed cluster state mast have a master. " + state; final ClusterStateContext pendingContext = pendingStates.get(index); final ClusterState pendingState = pendingContext.state; final DiscoveryNode pendingMasterNode = pendingState.nodes().getMasterNode(); if (Objects.equals(currentMaster, pendingMasterNode) == false) { contextsToRemove.add(pendingContext);
shardRouting.shardId(), state, nodes.getMasterNode()); if (nodes.getMasterNode() != null) { shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + state + "], mark shard as started", SHARD_STATE_ACTION_LISTENER, clusterState);
indexService = indicesService.createIndex(indexMetaData, buildInIndexListener); if (indexService.updateMapping(null, indexMetaData) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(), indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())
DiscoveryNode currentMasterNode = this.clusterState().getNodes().getMasterNode(); if (currentMasterNode == null) {
final DiscoveryNode masterNode = event.state().nodes().getMasterNode(); for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) { if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
try { if (indexService.updateMapping(currentIndexMetaData, newIndexMetaData) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(newIndexMetaData.getIndex().getName(), newIndexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())