private boolean shouldAcceptMastership(MemberMap memberMap, MemberImpl candidate) { assert lock.isHeldByCurrentThread() : "Called without holding cluster service lock!"; for (MemberImpl member : memberMap.headMemberSet(candidate, false)) { if (!membershipManager.isMemberSuspected(member.getAddress())) { if (logger.isFineEnabled()) { logger.fine("Should not accept mastership claim of " + candidate + ", because " + member + " is not suspected at the moment and is before than " + candidate + " in the member list."); } return false; } } return true; }
private boolean shouldAcceptMastership(MemberMap memberMap, MemberImpl candidate) { assert lock.isHeldByCurrentThread() : "Called without holding cluster service lock!"; for (MemberImpl member : memberMap.headMemberSet(candidate, false)) { if (!membershipManager.isMemberSuspected(member.getAddress())) { if (logger.isFineEnabled()) { logger.fine("Should not accept mastership claim of " + candidate + ", because " + member + " is not suspected at the moment and is before than " + candidate + " in the member list."); } return false; } } return true; }
private boolean shouldClaimMastership(MemberMap memberMap) { if (clusterService.isMaster()) { return false; } for (MemberImpl m : memberMap.headMemberSet(clusterService.getLocalMember(), false)) { if (!isMemberSuspected(m.getAddress())) { return false; } } return true; }
private boolean shouldClaimMastership(MemberMap memberMap) { if (clusterService.isMaster()) { return false; } for (MemberImpl m : memberMap.headMemberSet(clusterService.getLocalMember(), false)) { if (!isMemberSuspected(m.getAddress())) { return false; } } return true; }
} catch (TimeoutException ignored) { MemberInfo memberInfo = latestMembersView.getMember(address); if (mastershipClaimTimeout > 0 && !isMemberSuspected(address) && memberInfo != null) {
private boolean fetchMembersViewFromNewMembers(MembersView membersView, Map<Address, Future<MembersView>> futures) { boolean isNewMemberPresent = false; for (MemberInfo memberInfo : membersView.getMembers()) { Address memberAddress = memberInfo.getAddress(); if (!(node.getThisAddress().equals(memberAddress) || isMemberSuspected(memberAddress) || futures.containsKey(memberAddress))) { // this is a new member for us. lets ask its members view if (logger.isFineEnabled()) { logger.fine("Asking MembersView of " + memberAddress); } futures.put(memberAddress, invokeFetchMembersViewOp(memberAddress, memberInfo.getUuid())); isNewMemberPresent = true; } } return isNewMemberPresent; }
} catch (TimeoutException ignored) { MemberInfo memberInfo = latestMembersView.getMember(address); if (mastershipClaimTimeout > 0 && !isMemberSuspected(address) && memberInfo != null) {
private boolean fetchMembersViewFromNewMembers(MembersView membersView, Map<Address, Future<MembersView>> futures) { boolean isNewMemberPresent = false; for (MemberInfo memberInfo : membersView.getMembers()) { Address memberAddress = memberInfo.getAddress(); if (!(node.getThisAddress().equals(memberAddress) || isMemberSuspected(memberAddress) || futures.containsKey(memberAddress))) { // this is a new member for us. lets ask its members view if (logger.isFineEnabled()) { logger.fine("Asking MembersView of " + memberAddress); } futures.put(memberAddress, invokeFetchMembersViewOp(memberAddress, memberInfo.getUuid())); isNewMemberPresent = true; } } return isNewMemberPresent; }
if (isMemberSuspected(address)) { if (logger.isFineEnabled()) { logger.fine(memberInfo + " is excluded because suspected");
if (isMemberSuspected(address)) { if (logger.isFineEnabled()) { logger.fine(memberInfo + " is excluded because suspected");
/** * Removes the {@code member} if it has not sent any heartbeats in {@link GroupProperty#MAX_NO_HEARTBEAT_SECONDS}. * If it has not sent any heartbeats in {@link #HEART_BEAT_INTERVAL_FACTOR} heartbeat intervals, it will log a warning. * * @param now the current cluster clock time * @param member the member which needs to be checked * @return if the member has been removed */ private boolean suspectMemberIfNotHeartBeating(long now, Member member) { if (clusterService.getMembershipManager().isMemberSuspected(member.getAddress())) { return true; } long lastHeartbeat = heartbeatFailureDetector.lastHeartbeat(member); if (!heartbeatFailureDetector.isAlive(member, now)) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); String reason = format("Suspecting %s because it has not sent any heartbeats since %s." + " Now: %s, heartbeat timeout: %d ms, suspicion level: %.2f", member, timeToString(lastHeartbeat), timeToString(now), maxNoHeartbeatMillis, suspicionLevel); logger.warning(reason); clusterService.suspectMember(member, reason, true); return true; } if (logger.isFineEnabled() && (now - lastHeartbeat) > heartbeatIntervalMillis * HEART_BEAT_INTERVAL_FACTOR) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); logger.fine(format("Not receiving any heartbeats from %s since %s, suspicion level: %.2f", member, timeToString(lastHeartbeat), suspicionLevel)); } return false; }
/** * Removes the {@code member} if it has not sent any heartbeats in {@link GroupProperty#MAX_NO_HEARTBEAT_SECONDS}. * If it has not sent any heartbeats in {@link #HEART_BEAT_INTERVAL_FACTOR} heartbeat intervals, it will log a warning. * * @param now the current cluster clock time * @param member the member which needs to be checked * @return if the member has been removed */ private boolean suspectMemberIfNotHeartBeating(long now, Member member) { if (clusterService.getMembershipManager().isMemberSuspected(member.getAddress())) { return true; } long lastHeartbeat = heartbeatFailureDetector.lastHeartbeat(member); if (!heartbeatFailureDetector.isAlive(member, now)) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); String reason = format("Suspecting %s because it has not sent any heartbeats since %s." + " Now: %s, heartbeat timeout: %d ms, suspicion level: %.2f", member, timeToString(lastHeartbeat), timeToString(now), maxNoHeartbeatMillis, suspicionLevel); logger.warning(reason); clusterService.suspectMember(member, reason, true); return true; } if (logger.isFineEnabled() && (now - lastHeartbeat) > heartbeatIntervalMillis * HEART_BEAT_INTERVAL_FACTOR) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); logger.fine(format("Not receiving any heartbeats from %s since %s, suspicion level: %.2f", member, timeToString(lastHeartbeat), suspicionLevel)); } return false; }
/** * Sends heartbeat to each of the cluster members. * Checks whether the master member has failed to send a heartbeat (see {@link #maxNoHeartbeatMillis}) * and removes that master member from cluster, if it fails on heartbeat. * <p></p> * This method is called on NON-master members. */ private void heartbeatWhenSlave(long now) { MembershipManager membershipManager = clusterService.getMembershipManager(); Collection<Member> members = clusterService.getMembers(MemberSelectors.NON_LOCAL_MEMBER_SELECTOR); for (Member member : members) { try { logIfConnectionToEndpointIsMissing(now, member); if (suspectMemberIfNotHeartBeating(now, member)) { continue; } if (membershipManager.isMemberSuspected(member.getAddress())) { continue; } pingMemberIfRequired(now, member); sendHeartbeat(member); } catch (Throwable e) { logger.severe(e); } } }
/** * Sends heartbeat to each of the cluster members. * Checks whether the master member has failed to send a heartbeat (see {@link #maxNoHeartbeatMillis}) * and removes that master member from cluster, if it fails on heartbeat. * <p></p> * This method is called on NON-master members. */ private void heartbeatWhenSlave(long now) { MembershipManager membershipManager = clusterService.getMembershipManager(); Collection<Member> members = clusterService.getMembers(MemberSelectors.NON_LOCAL_MEMBER_SELECTOR); for (Member member : members) { try { logIfConnectionToEndpointIsMissing(now, member); if (suspectMemberIfNotHeartBeating(now, member)) { continue; } if (membershipManager.isMemberSuspected(member.getAddress())) { continue; } pingMemberIfRequired(now, member); sendHeartbeat(member); } catch (Throwable e) { logger.severe(e); } } }
private void onInvalidHeartbeat(MembersViewMetadata senderMembersViewMetadata) { Address senderAddress = senderMembersViewMetadata.getMemberAddress(); if (clusterService.isMaster()) { if (!clusterService.getClusterJoinManager().isMastershipClaimInProgress()) { logger.fine("Sending explicit suspicion to " + senderAddress + " for heartbeat " + senderMembersViewMetadata + ", because it is not a member of this cluster" + " or its heartbeat cannot be validated!"); clusterService.sendExplicitSuspicion(senderMembersViewMetadata); } } else { Address masterAddress = clusterService.getMasterAddress(); if (clusterService.getMembershipManager().isMemberSuspected(masterAddress)) { logger.fine("Not sending heartbeat complaint for " + senderMembersViewMetadata + " to suspected master: " + masterAddress); return; } logger.fine("Sending heartbeat complaint to master " + masterAddress + " for heartbeat " + senderMembersViewMetadata + ", because it is not a member of this cluster" + " or its heartbeat cannot be validated!"); sendHeartbeatComplaintToMaster(senderMembersViewMetadata); } }
private void onInvalidHeartbeat(MembersViewMetadata senderMembersViewMetadata) { Address senderAddress = senderMembersViewMetadata.getMemberAddress(); if (clusterService.isMaster()) { if (!clusterService.getClusterJoinManager().isMastershipClaimInProgress()) { logger.fine("Sending explicit suspicion to " + senderAddress + " for heartbeat " + senderMembersViewMetadata + ", because it is not a member of this cluster" + " or its heartbeat cannot be validated!"); clusterService.sendExplicitSuspicion(senderMembersViewMetadata); } } else { Address masterAddress = clusterService.getMasterAddress(); if (clusterService.getMembershipManager().isMemberSuspected(masterAddress)) { logger.fine("Not sending heartbeat complaint for " + senderMembersViewMetadata + " to suspected master: " + masterAddress); return; } logger.fine("Sending heartbeat complaint to master " + masterAddress + " for heartbeat " + senderMembersViewMetadata + ", because it is not a member of this cluster" + " or its heartbeat cannot be validated!"); sendHeartbeatComplaintToMaster(senderMembersViewMetadata); } }