@Override public void afterRun() throws Exception { if (suspectedCaller != null) { ClusterServiceImpl service = getService(); // I am the master. I can remove the member directly String reason = "Removing " + suspectedCaller + ", since it thinks it's already split from this cluster " + "and looking to merge."; service.suspectMember(suspectedCaller, reason, true); } }
@Override public void afterRun() throws Exception { if (suspectedCaller != null) { ClusterServiceImpl service = getService(); // I am the master. I can remove the member directly String reason = "Removing " + suspectedCaller + ", since it thinks it's already split from this cluster " + "and looking to merge."; service.suspectMember(suspectedCaller, reason, true); } }
private static void closeConnectionBetween(HazelcastInstance h1, HazelcastInstance h2) { if (h1 == null || h2 == null) { return; } Node n1 = getNode(h1); Node n2 = getNode(h2); if (n1 != null && n2 != null) { n1.clusterService.suspectMember(n2.getLocalMember(), null, true); n2.clusterService.suspectMember(n1.getLocalMember(), null, true); } } }
public void run() { Address address = member.getAddress(); logger.warning(format("%s will ping %s", node.getThisAddress(), address)); for (int i = 0; i < MAX_PING_RETRY_COUNT; i++) { if (doPing(address, Level.INFO)) { return; } } // host not reachable String reason = format("%s could not ping %s", node.getThisAddress(), address); logger.warning(reason); clusterService.suspectMember(member, reason, true); }
public void run() { Address address = member.getAddress(); logger.warning(format("%s will ping %s", node.getThisAddress(), address)); for (int i = 0; i < MAX_PING_RETRY_COUNT; i++) { if (doPing(address, Level.INFO)) { return; } } // host not reachable String reason = format("%s could not ping %s", node.getThisAddress(), address); logger.warning(reason); clusterService.suspectMember(member, reason, true); }
public void suspectAddressIfNotConnected(Address address) { lock.lock(); try { MemberImpl member = getMember(address); if (member == null) { if (logger.isFineEnabled()) { logger.fine("Cannot suspect " + address + ", since it's not a member."); } return; } Connection conn = node.getConnectionManager().getConnection(address); if (conn != null && conn.isAlive()) { if (logger.isFineEnabled()) { logger.fine("Cannot suspect " + member + ", since there's a live connection -> " + conn); } return; } suspectMember(member, "No connection", false); } finally { lock.unlock(); } }
public void suspectAddressIfNotConnected(Address address) { lock.lock(); try { MemberImpl member = getMember(address); if (member == null) { if (logger.isFineEnabled()) { logger.fine("Cannot suspect " + address + ", since it's not a member."); } return; } Connection conn = node.getConnectionManager().getConnection(address); if (conn != null && conn.isAlive()) { if (logger.isFineEnabled()) { logger.fine("Cannot suspect " + member + ", since there's a live connection -> " + conn); } return; } suspectMember(member, "No connection", false); } finally { lock.unlock(); } }
public void run() { Address address = member.getAddress(); logger.fine(format("%s will ping %s", node.getThisAddress(), address)); if (doPing(address, Level.FINE)) { boolean pingRestored = (icmpFailureDetector.heartbeat(member) > 0); if (pingRestored) { quorumService.onPingRestored(member); } return; } icmpFailureDetector.logAttempt(member); quorumService.onPingLost(member); // host not reachable String reason = format("%s could not ping %s", node.getThisAddress(), address); logger.warning(reason); if (!icmpFailureDetector.isAlive(member)) { clusterService.suspectMember(member, reason, true); } } }
public void run() { Address address = member.getAddress(); logger.fine(format("%s will ping %s", node.getThisAddress(), address)); if (doPing(address, Level.FINE)) { boolean pingRestored = (icmpFailureDetector.heartbeat(member) > 0); if (pingRestored) { quorumService.onPingRestored(member); } return; } icmpFailureDetector.logAttempt(member); quorumService.onPingLost(member); // host not reachable String reason = format("%s could not ping %s", node.getThisAddress(), address); logger.warning(reason); if (!icmpFailureDetector.isAlive(member)) { clusterService.suspectMember(member, reason, true); } } }
/** * Removes the {@code member} if it has not sent any heartbeats in {@link GroupProperty#MAX_NO_HEARTBEAT_SECONDS}. * If it has not sent any heartbeats in {@link #HEART_BEAT_INTERVAL_FACTOR} heartbeat intervals, it will log a warning. * * @param now the current cluster clock time * @param member the member which needs to be checked * @return if the member has been removed */ private boolean suspectMemberIfNotHeartBeating(long now, Member member) { if (clusterService.getMembershipManager().isMemberSuspected(member.getAddress())) { return true; } long lastHeartbeat = heartbeatFailureDetector.lastHeartbeat(member); if (!heartbeatFailureDetector.isAlive(member, now)) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); String reason = format("Suspecting %s because it has not sent any heartbeats since %s." + " Now: %s, heartbeat timeout: %d ms, suspicion level: %.2f", member, timeToString(lastHeartbeat), timeToString(now), maxNoHeartbeatMillis, suspicionLevel); logger.warning(reason); clusterService.suspectMember(member, reason, true); return true; } if (logger.isFineEnabled() && (now - lastHeartbeat) > heartbeatIntervalMillis * HEART_BEAT_INTERVAL_FACTOR) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); logger.fine(format("Not receiving any heartbeats from %s since %s, suspicion level: %.2f", member, timeToString(lastHeartbeat), suspicionLevel)); } return false; }
/** * Removes the {@code member} if it has not sent any heartbeats in {@link GroupProperty#MAX_NO_HEARTBEAT_SECONDS}. * If it has not sent any heartbeats in {@link #HEART_BEAT_INTERVAL_FACTOR} heartbeat intervals, it will log a warning. * * @param now the current cluster clock time * @param member the member which needs to be checked * @return if the member has been removed */ private boolean suspectMemberIfNotHeartBeating(long now, Member member) { if (clusterService.getMembershipManager().isMemberSuspected(member.getAddress())) { return true; } long lastHeartbeat = heartbeatFailureDetector.lastHeartbeat(member); if (!heartbeatFailureDetector.isAlive(member, now)) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); String reason = format("Suspecting %s because it has not sent any heartbeats since %s." + " Now: %s, heartbeat timeout: %d ms, suspicion level: %.2f", member, timeToString(lastHeartbeat), timeToString(now), maxNoHeartbeatMillis, suspicionLevel); logger.warning(reason); clusterService.suspectMember(member, reason, true); return true; } if (logger.isFineEnabled() && (now - lastHeartbeat) > heartbeatIntervalMillis * HEART_BEAT_INTERVAL_FACTOR) { double suspicionLevel = heartbeatFailureDetector.suspicionLevel(member, now); logger.fine(format("Not receiving any heartbeats from %s since %s, suspicion level: %.2f", member, timeToString(lastHeartbeat), suspicionLevel)); } return false; }
private void handleActiveAndJoined(JoinMessage joinMessage) { if (!(joinMessage instanceof JoinRequest)) { logDroppedMessage(joinMessage); return; } ClusterServiceImpl clusterService = node.getClusterService(); Address masterAddress = clusterService.getMasterAddress(); if (clusterService.isMaster()) { JoinMessage response = new JoinMessage(Packet.VERSION, node.getBuildInfo().getBuildNumber(), node.getVersion(), node.getThisAddress(), node.getThisUuid(), node.isLiteMember(), node.createConfigCheck()); node.multicastService.send(response); } else if (joinMessage.getAddress().equals(masterAddress)) { MemberImpl master = node.getClusterService().getMember(masterAddress); if (master != null && !master.getUuid().equals(joinMessage.getUuid())) { String message = "New join request has been received from current master. Suspecting " + masterAddress; logger.warning(message); // I just make a local suspicion. Probably other nodes will eventually suspect as well. clusterService.suspectMember(master, message, false); } } }
private void handleActiveAndJoined(JoinMessage joinMessage) { if (!(joinMessage instanceof JoinRequest)) { logDroppedMessage(joinMessage); return; } ClusterServiceImpl clusterService = node.getClusterService(); Address masterAddress = clusterService.getMasterAddress(); if (clusterService.isMaster()) { JoinMessage response = new JoinMessage(Packet.VERSION, node.getBuildInfo().getBuildNumber(), node.getVersion(), node.getThisAddress(), node.getThisUuid(), node.isLiteMember(), node.createConfigCheck()); node.multicastService.send(response); } else if (joinMessage.getAddress().equals(masterAddress)) { MemberImpl master = node.getClusterService().getMember(masterAddress); if (master != null && !master.getUuid().equals(joinMessage.getUuid())) { String message = "New join request has been received from current master. Suspecting " + masterAddress; logger.warning(message); // I just make a local suspicion. Probably other nodes will eventually suspect as well. clusterService.suspectMember(master, message, false); } } }
logger.warning(msg); clusterService.suspectMember(member, msg, false); Connection existing = node.connectionManager.getConnection(target); if (existing != connection) {
logger.warning(msg); clusterService.suspectMember(member, msg, false); Connection existing = node.connectionManager.getConnection(target); if (existing != connection) {