public void onRestart(InetAddress endpoint, EndpointState state) { // If we have restarted before the node was even marked down, we need to reset the connection pool if (state.isAlive()) onDead(endpoint, state); // Then, the node may have been upgraded and changed its messaging protocol version. If so, we // want to update that before we mark the node live again to avoid problems like CASSANDRA-11128. VersionedValue netVersion = state.getApplicationState(ApplicationState.NET_VERSION); if (netVersion != null) updateNetVersion(endpoint, netVersion); }
@VisibleForTesting public void initializeNodeUnsafe(InetAddress addr, UUID uuid, int generationNbr) { HeartBeatState hbState = new HeartBeatState(generationNbr); EndpointState newState = new EndpointState(hbState); newState.markAlive(); EndpointState oldState = endpointStateMap.putIfAbsent(addr, newState); EndpointState localState = oldState == null ? newState : oldState; // always add the version state Map<ApplicationState, VersionedValue> states = new EnumMap<>(ApplicationState.class); states.put(ApplicationState.NET_VERSION, StorageService.instance.valueFactory.networkVersion()); states.put(ApplicationState.HOST_ID, StorageService.instance.valueFactory.hostId(uuid)); localState.addApplicationStates(states); }
@VisibleForTesting public void injectApplicationState(InetAddress endpoint, ApplicationState state, VersionedValue value) { EndpointState localState = endpointStateMap.get(endpoint); localState.addApplicationState(state, value); }
private void applyNewStates(InetAddress addr, EndpointState localState, EndpointState remoteState) { // don't assert here, since if the node restarts the version will go back to zero int oldVersion = localState.getHeartBeatState().getHeartBeatVersion(); localState.setHeartBeatState(remoteState.getHeartBeatState()); if (logger.isTraceEnabled()) logger.trace("Updating heartbeat state version to {} from {} for {} ...", localState.getHeartBeatState().getHeartBeatVersion(), oldVersion, addr); Set<Entry<ApplicationState, VersionedValue>> remoteStates = remoteState.states(); assert remoteState.getHeartBeatState().getGeneration() == localState.getHeartBeatState().getGeneration(); localState.addApplicationStates(remoteStates); for (Entry<ApplicationState, VersionedValue> remoteEntry : remoteStates) doOnChangeNotifications(addr, remoteEntry.getKey(), remoteEntry.getValue()); }
/** * Return either: the greatest heartbeat or application state * * @param epState * @return */ int getMaxEndpointStateVersion(EndpointState epState) { int maxVersion = epState.getHeartBeatState().getHeartBeatVersion(); for (Map.Entry<ApplicationState, VersionedValue> state : epState.states()) maxVersion = Math.max(maxVersion, state.getValue().version); return maxVersion; }
/** * Add an endpoint we knew about previously, but whose state is unknown */ public void addSavedEndpoint(InetAddress ep) { if (ep.equals(FBUtilities.getBroadcastAddress())) { logger.debug("Attempt to add self as saved endpoint"); return; } //preserve any previously known, in-memory data about the endpoint (such as DC, RACK, and so on) EndpointState epState = endpointStateMap.get(ep); if (epState != null) { logger.debug("not replacing a previous epState for {}, but reusing it: {}", ep, epState); epState.setHeartBeatState(new HeartBeatState(0)); } else { epState = new EndpointState(new HeartBeatState(0)); } epState.markDead(); endpointStateMap.put(ep, epState); unreachableEndpoints.put(ep, System.nanoTime()); if (logger.isTraceEnabled()) logger.trace("Adding saved endpoint {} {}", ep, epState.getHeartBeatState().getGeneration()); }
VersionedValue vv = state.getApplicationState(ApplicationState.HOST_ID); if (vv != null) { String hostId = vv.value; attrs.put("rack", DatabaseDescriptor.getEndpointSnitch().getRack(endpoint)); InetAddress internal_address = com.google.common.net.InetAddresses.forString(state.getApplicationState(ApplicationState.INTERNAL_IP).value); dn = new DiscoveryNode(buildNodeName(internal_address), hostId.toString(), new InetSocketTransportAddress(internal_address, publishPort()), attrs, CASSANDRA_ROLES, Version.CURRENT); NetworkAddress.format(endpoint), dn.getId(), dn.getName(), state.isAlive(), state.getStatus(), state.getUpdateTimestamp()); if (ElassandraDaemon.hasWorkloadColumn && (state.getApplicationState(ApplicationState.X1) != null || state.getApplicationState(ApplicationState.X2) !=null)) { SystemKeyspace.updatePeerInfo(endpoint, "workload", "elasticsearch", StageManager.getStage(Stage.MUTATION)); if (state.getApplicationState(ApplicationState.X1) != null) { VersionedValue x1 = state.getApplicationState(ApplicationState.X1); if (!endpoint.equals(this.localAddress)) { Map<String, ShardRoutingState> shardsStateMap; if (state.getApplicationState(ApplicationState.X2) != null) { highestVersionSeen = Long.max(highestVersionSeen, getMetadataVersion(state.getApplicationState(ApplicationState.X2)));
&& TimeUnit.NANOSECONDS.toMillis(nowNano - epState.getUpdateTimestamp()) > fatClientTimeout) if (!epState.isAlive() && (now > expireTime) && (!StorageService.instance.getTokenMetadata().isMember(endpoint)))
return; String hostId = endPointState.getApplicationState(ApplicationState.HOST_ID).value; if (hostId == null || this.localNode.getId().equals(hostId) || !listener.attendees.containsKey(hostId)) return; if (!endPointState.isAlive() || !endPointState.getStatus().equals("NORMAL")) { VersionedValue vv = endPointState.getApplicationState(ApplicationState.X2); if (vv != null && getMetadataVersion(vv) >= listener.version()) { DiscoveryNode node = listener.attendees.remove(hostId);
/** * Handles switching the endpoint's state from REMOVING_TOKEN to REMOVED_TOKEN * This should only be called after advertiseRemoving * * @param endpoint * @param hostId */ public void advertiseTokenRemoved(InetAddress endpoint, UUID hostId) { EndpointState epState = endpointStateMap.get(endpoint); epState.updateTimestamp(); // make sure we don't evict it too soon epState.getHeartBeatState().forceNewerGenerationUnsafe(); long expireTime = computeExpireTime(); epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.removedNonlocal(hostId, expireTime)); logger.info("Completing removal of {}", endpoint); addExpireTimeForEndpoint(endpoint, expireTime); endpointStateMap.put(endpoint, epState); // ensure at least one gossip round occurs before returning Uninterruptibles.sleepUninterruptibly(intervalInMillis * 2, TimeUnit.MILLISECONDS); }
private static String getGossipStatus(EndpointState epState) { if (epState == null || epState.getApplicationState(ApplicationState.STATUS) == null) return ""; String value = epState.getApplicationState(ApplicationState.STATUS).value; String[] pieces = value.split(VersionedValue.DELIMITER_STR, -1); assert (pieces.length > 0); return pieces[0]; }
public void maybeInitializeLocalState(int generationNbr) { HeartBeatState hbState = new HeartBeatState(generationNbr); EndpointState localState = new EndpointState(hbState); localState.markAlive(); endpointStateMap.putIfAbsent(FBUtilities.getBroadcastAddress(), localState); }
/** * This method is used to mark a node as shutdown; that is it gracefully exited on its own and told us about it * @param endpoint endpoint that has shut itself down */ protected void markAsShutdown(InetAddress endpoint) { EndpointState epState = endpointStateMap.get(endpoint); if (epState == null) return; epState.addApplicationState(ApplicationState.STATUS, StorageService.instance.valueFactory.shutdown(true)); epState.addApplicationState(ApplicationState.RPC_READY, StorageService.instance.valueFactory.rpcReady(false)); epState.getHeartBeatState().forceHighestPossibleVersionUnsafe(); markDead(endpoint, epState); FailureDetector.instance.forceConviction(endpoint); }
@VisibleForTesting public void realMarkAlive(final InetAddress addr, final EndpointState localState) { if (logger.isTraceEnabled()) logger.trace("marking as alive {}", addr); localState.markAlive(); localState.updateTimestamp(); // prevents doStatusCheck from racing us and evicting if it was down > aVeryLongTime liveEndpoints.add(addr); unreachableEndpoints.remove(addr); expireTimeEndpointMap.remove(addr); logger.debug("removing expire time for endpoint : {}", addr); logger.info("InetAddress {} is now UP", addr); for (IEndpointStateChangeSubscriber subscriber : subscribers) subscriber.onAlive(addr, localState); if (logger.isTraceEnabled()) logger.trace("Notified {}", subscribers); }
public int getDownEndpointCount() { int count = 0; for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.endpointStateMap.entrySet()) { if (!entry.getValue().isAlive()) count++; } return count; }
public void updateNode(InetAddress endpoint, EndpointState state, DiscoveryNodeStatus newStatus) { if (isLocal(endpoint)) { boolean updatedNode = false; String hostId = state.getApplicationState(ApplicationState.HOST_ID).value; DiscoveryNode dn = clusterGroup.get(hostId); if (dn == null) { attrs.put("rack", DatabaseDescriptor.getEndpointSnitch().getRack(endpoint)); InetAddress internal_address = com.google.common.net.InetAddresses.forString(state.getApplicationState(ApplicationState.INTERNAL_IP).value); dn = new DiscoveryNode(buildNodeName(internal_address), hostId.toString(), dn.status(newStatus); logger.debug("New node soure=updateNode internal_ip={} rpc_address={}, node_name={} host_id={} status={} timestamp={}", NetworkAddress.format(endpoint), NetworkAddress.format(internal_address), dn.getId(), dn.getName(), newStatus, state.getUpdateTimestamp()); clusterGroup.members.put(dn.getId(), dn); VersionedValue x1 = state.getApplicationState(ApplicationState.X1); if (x1 != null) { try {
public int getCurrentGenerationNumber(InetAddress endpoint) { return endpointStateMap.get(endpoint).getHeartBeatState().getGeneration(); }
@VisibleForTesting public void markDead(InetAddress addr, EndpointState localState) { if (logger.isTraceEnabled()) logger.trace("marking as down {}", addr); localState.markDead(); liveEndpoints.remove(addr); unreachableEndpoints.put(addr, System.nanoTime()); logger.info("InetAddress {} is now DOWN", addr); for (IEndpointStateChangeSubscriber subscriber : subscribers) subscriber.onDead(addr, localState); if (logger.isTraceEnabled()) logger.trace("Notified {}", subscribers); }
public void addApplicationState(ApplicationState key, VersionedValue value) { addApplicationStates(Collections.singletonMap(key, value)); }