/** * Transfer the failure-detection ports from another view to this one */ public void setFailureDetectionPorts(NetView otherView) { int[] ports = otherView.getFailureDetectionPorts(); if (ports != null) { int idx = 0; int portsSize = ports.length; for (InternalDistributedMember mbr : otherView.getMembers()) { if (contains(mbr)) { // unit tests create views w/o failure detection ports, so we must check the length // of the array if (idx < portsSize) { setFailureDetectionPort(mbr, ports[idx]); } else { setFailureDetectionPort(mbr, -1); } } idx += 1; } } }
@Override public void installView(NetView v) { this.view = v; if (this.jgAddress.getVmViewId() < 0) { this.jgAddress.setVmViewId(this.localAddress.getVmViewId()); } List<JGAddress> mbrs = new ArrayList<>(v.size()); mbrs.addAll(v.getMembers().stream().map(JGAddress::new).collect(Collectors.toList())); ViewId vid = new ViewId(new JGAddress(v.getCoordinator()), v.getViewId()); View jgv = new View(vid, new ArrayList<>(mbrs)); logger.trace("installing view into JGroups stack: {}", jgv); this.myChannel.down(new Event(Event.VIEW_CHANGE, jgv)); addressesWithIoExceptionsProcessed.clear(); if (encrypt != null) { encrypt.installView(v); } synchronized (scheduledMcastSeqnos) { for (DistributedMember mbr : v.getCrashedMembers()) { scheduledMcastSeqnos.remove(mbr); } for (DistributedMember mbr : v.getShutdownMembers()) { scheduledMcastSeqnos.remove(mbr); } } }
/** * check to see if the new view shows a drop of 51% or more */ private boolean isNetworkPartition(NetView newView, boolean logWeights) { if (currentView == null) { return false; } int oldWeight = currentView.memberWeight(); int failedWeight = newView.getCrashedMemberWeight(currentView); if (failedWeight > 0 && logWeights) { if (logger.isInfoEnabled() && newView.getCreator().equals(localAddress)) { // view-creator // logs this newView.logCrashedMemberWeights(currentView, logger); } int failurePoint = (int) (Math.round(51.0 * oldWeight) / 100.0); if (failedWeight > failurePoint && quorumLostView != newView) { quorumLostView = newView; logger.warn("total weight lost in this view change is {} of {}. Quorum has been lost!", failedWeight, oldWeight); services.getManager().quorumLost(newView.getActualCrashedMembers(currentView), currentView); return true; } } return false; }
/** * Returns a copy (possibly not current) of the current view (a list of * {@link DistributedMember}s) */ @Override public NetView getView() { // Grab the latest view under a mutex... NetView v; latestViewReadLock.lock(); v = latestView; latestViewReadLock.unlock(); NetView result = new NetView(v, v.getViewId()); v.getMembers().stream().filter(this::isShunned).forEachOrdered(result::remove); return result; }
if (currentView == null || currentView.getViewId() < v.getViewId()) { int viewId = Math.max(initialView.getViewId(), v.getViewId()); viewId += 1; NetView newView = new NetView(initialView, viewId); newMembers = v.getNewMembers(currentView); } else { newMembers = v.getMembers(); newView.add(newMember); newView.setFailureDetectionPort(newMember, v.getFailureDetectionPort(newMember)); newView.setPublicKey(newMember, v.getPublicKey(newMember));
@Test public void testCreateView() throws Exception { int numMembers = members.size(); NetView view = new NetView(members.get(0), 2, members); setFailureDetectionPorts(view); assertTrue(view.getCreator().equals(members.get(0))); assertEquals(2, view.getViewId()); assertEquals(members, view.getMembers()); assertEquals(0, view.getCrashedMembers().size()); assertEquals(members.get(1), view.getLeadMember()); // a locator can't be lead member assertEquals(0, view.getShutdownMembers().size()); assertEquals(1, view.getNewMembers().size()); assertEquals(members.get(numMembers - 1), view.getNewMembers().iterator().next()); assertEquals(members.get(0), view.getCoordinator()); for (int i = 0; i < numMembers; i++) { InternalDistributedMember mbr = members.get(i); assertEquals(mbr.getPort(), view.getFailureDetectionPort(mbr)); } assertFalse(view.shouldBeCoordinator(members.get(1))); assertTrue(view.shouldBeCoordinator(members.get(0))); assertEquals(members.get(numMembers - 1), view.getCoordinator(Collections.singletonList(members.get(0)))); members.get(numMembers - 1).getNetMember().setPreferredForCoordinator(false); assertEquals(members.get(1), view.getCoordinator(Collections.singletonList(members.get(0)))); members.get(numMembers - 1).getNetMember().setPreferredForCoordinator(true); List<InternalDistributedMember> preferred = view.getPreferredCoordinators( Collections.<InternalDistributedMember>singleton(members.get(1)), members.get(0), 2); assertEquals(2, preferred.size()); assertEquals(members.get(numMembers - 1), preferred.get(0)); }
Set<InternalDistributedMember> crashes = newView.getActualCrashedMembers(currentView); forceDisconnect(String.format( "Exiting due to possible network partition event due to loss of %s cache processes: %s", new ArrayList<>(currentView.getCrashedMembers().size() + unresponsive.size()); conflictingView != null && !conflictingView.getCreator().equals(localAddress) && conflictingView.getViewId() > newView.getViewId(); if (conflictingViewNotFromMe) { boolean conflictingViewIsMostRecent = (lastConflictingView == null || conflictingView.getViewId() > lastConflictingView.getViewId()); if (conflictingViewIsMostRecent) { lastConflictingView = conflictingView; && conflictingView.getCreator() .getVmKind() == ClusterDistributionManager.LOCATOR_DM_TYPE) { logger.info("View preparation interrupted - a locator is taking over as " failures.addAll(conflictingView.getCrashedMembers()); List<InternalDistributedMember> newMembers = conflictingView.getNewMembers(); if (!newMembers.isEmpty()) { logger.info("adding these new members from a conflicting view to the new view: {}", newMembers); for (InternalDistributedMember mbr : newMembers) { int port = conflictingView.getFailureDetectionPort(mbr); newView.add(mbr); newView.setFailureDetectionPort(mbr, port); joinReqs.add(mbr);
@Test public void testCopyView() throws Exception { NetView view = new NetView(members.get(0), 2, new ArrayList<>(members)); setFailureDetectionPorts(view); NetView newView = new NetView(view, 3); assertTrue(newView.getCreator().equals(members.get(0))); assertEquals(3, newView.getViewId()); assertEquals(members, newView.getMembers()); assertEquals(0, newView.getCrashedMembers().size()); assertEquals(members.get(1), newView.getLeadMember()); // a locator can't be lead member assertEquals(0, newView.getShutdownMembers().size()); assertEquals(0, newView.getNewMembers().size()); assertTrue(newView.equals(view)); assertTrue(view.equals(newView)); newView.remove(members.get(1)); assertFalse(newView.equals(view)); }
if (currentView != null && currentView.getViewId() >= newView.getViewId()) { for (InternalDistributedMember mbr : newView.getMembers()) { if (this.localAddress.equals(mbr)) { found = true; Set<InternalDistributedMember> crashes = newView.getActualCrashedMembers(currentView); forceDisconnect(String.format( "Exiting due to possible network partition event due to loss of %s cache processes: %s", if (!newView.getCreator().equals(this.localAddress)) { NetView check = new NetView(newView, newView.getViewId() + 1); synchronized (leftMembers) { check.removeAll(leftMembers); check.removeAll(removedMembers); check.addCrashedMembers(removedMembers); if (check.shouldBeCoordinator(this.localAddress)) { if (!isCoordinator) { becomeCoordinator(); DistributionMessage m = it.next(); if (m instanceof JoinRequestMessage) { if (currentView.contains(((JoinRequestMessage) m).getMemberID())) { it.remove(); if (!currentView.contains(((LeaveRequestMessage) m).getMemberID())) { it.remove();
+ services.getCancelCriterion().isCancelInProgress()); if (!v.contains(mbr) && mbr.getVmViewId() < v.getViewId()) { logger.info("ignoring leave request from old member"); return; NetView check = new NetView(v, v.getViewId() + 1); check.remove(mbr); synchronized (removedMembers) { check.removeAll(removedMembers); check.addCrashedMembers(removedMembers); check.removeAll(leftMembers); if (check.getCoordinator().equals(localAddress)) { synchronized (viewInstallationLock) { becomeCoordinator(mbr);
synchronized (viewInstallationLock) { int rand = testing ? 0 : NetView.RANDOM.nextInt(10); int viewNumber = currentView.getViewId() + 5 + rand; if (this.localAddress.getVmViewId() < 0) { this.localAddress.setVmViewId(viewNumber); List<InternalDistributedMember> mbrs = new ArrayList<>(currentView.getMembers()); if (!mbrs.contains(localAddress)) { mbrs.add(localAddress); newView = new NetView(this.localAddress, viewNumber, mbrs, leaving, removals); newView.setFailureDetectionPorts(currentView); newView.setPublicKeys(currentView); newView.setFailureDetectionPort(this.localAddress, services.getHealthMonitor().getFailureDetectionPort());
cv == null ? "<no view>" : cv.getViewId()); if (cv.getViewId() >= viewId && !cv.contains(incomingRequest.getSender())) { logger.info("Membership ignoring suspect request for " + incomingRequest + " from non-member " + incomingRequest.getSender()); suspectRequests, cv, cv.getCoordinator()); if (cv.getCoordinator().equals(localAddress)) { NetView check = new NetView(cv, cv.getViewId() + 1); ArrayList<SuspectRequest> membersToCheck = new ArrayList<>(); synchronized (suspectRequestsInView) { logger.info("Current suspects are {}", suspectsInView); for (final SuspectRequest sr : suspectsInView) { check.remove(sr.getSuspectMember()); membersToCheck.add(sr); for (InternalDistributedMember member : cv.getMembers()) { if (services.getJoinLeave().isMemberLeaving(member)) { membersLeaving.add(member); check.removeAll(membersLeaving); check.getCoordinator(), localAddress); InternalDistributedMember coordinator = check.getCoordinator(); if (coordinator != null && coordinator.equals(localAddress)) {
msg.append("} on ").append(address.toString()); logger.debug(msg); if (!newView.contains(address)) { logger.info("The Member with id {}, is no longer in my own view, {}", address, newView); for (InternalDistributedMember mbr : newView.getMembers()) { Version itsVersion = mbr.getVersionObject(); if (itsVersion != null && itsVersion.compareTo(version) < 0) { latestView = new NetView(newView, newView.getViewId()); for (int i = 0; i < newView.getMembers().size(); i++) { // additions InternalDistributedMember m = newView.getMembers().get(i); if (priorView.contains(m) || wasSurprise) { continue; // already seen for (int i = 0; i < priorView.getMembers().size(); i++) { // departures InternalDistributedMember m = priorView.getMembers().get(i); if (newView.contains(m)) { continue; // still alive newView.getCrashedMembers().contains(m) || suspectedMembers.containsKey(m), "departed membership view"); } catch (VirtualMachineError err) { "not seen in membership view in " + this.surpriseMemberTimeout + "ms"); } else {
@Test public void testAddLotsOfMembers() throws Exception { NetView view = new NetView(members.get(0), 2, new ArrayList<>(members)); setFailureDetectionPorts(view); NetView copy = new NetView(view, 2); int oldSize = view.size(); for (int i = 0; i < 100; i++) { InternalDistributedMember mbr = new InternalDistributedMember(SocketCreator.getLocalHost(), 2000 + i); mbr.setVmKind(ClusterDistributionManager.NORMAL_DM_TYPE); mbr.setVmViewId(2); view.add(mbr); view.setFailureDetectionPort(mbr, 2000 + i); } assertEquals(oldSize + 100, view.size()); for (InternalDistributedMember mbr : view.getMembers()) { assertEquals(mbr.getPort(), view.getFailureDetectionPort(mbr)); } assertEquals(100, view.getNewMembers(copy).size()); }
for (InternalDistributedMember id : v.getMembers()) { if (requestingMemberID.compareTo(id, false) == 0) { NetView newView = new NetView(v, v.getViewId()); newView.remove(id); v = newView; break; if (v.getViewId() > findRequest.getLastViewId()) { coordinator = v.getCoordinator(Collections.emptyList()); } else { coordinator = v.getCoordinator(findRequest.getRejectedCoordinators()); coordinator = localAddress; if (v != null && localAddress != null && !localAddress.equals(v.getCoordinator())) { logger.info("This member is becoming coordinator since view {}", v); v = null; coordinatorPublicKey = (byte[]) v.getPublicKey(coordinator);
@Test public void testRemoveAll() throws Exception { int numMembers = members.size(); NetView view = new NetView(members.get(0), 2, new ArrayList<>(members)); setFailureDetectionPorts(view); Collection<InternalDistributedMember> removals = new ArrayList<>(numMembers / 2); for (int i = 1; i < numMembers; i += 2) { removals.add(members.get(i)); } view.removeAll(removals); for (InternalDistributedMember mbr : removals) { assertFalse(view.contains(mbr)); } assertEquals(numMembers - removals.size(), view.size()); List<InternalDistributedMember> remainingMembers = view.getMembers(); int num = remainingMembers.size(); for (int i = 0; i < num; i++) { InternalDistributedMember mbr = remainingMembers.get(i); assertEquals(mbr.getPort(), view.getFailureDetectionPort(mbr)); } }
private boolean sendView(NetView view, boolean preparing, ViewReplyProcessor viewReplyProcessor) throws InterruptedException { int id = view.getViewId(); InstallViewMessage msg = new InstallViewMessage(view, services.getAuthenticator().getCredentials(this.localAddress), preparing); Set<InternalDistributedMember> recips = new HashSet<>(view.getMembers()); if (!view.getCrashedMembers().isEmpty()) { recips = new HashSet<>(recips); recips.addAll(view.getCrashedMembers()); if (isCoordinator && !localAddress.equals(view.getCoordinator()) && getViewCreator() != null) { getViewCreator().markViewCreatorForShutdown(); this.isCoordinator = false; installView(new NetView(view, view.getViewId())); pendingRemovals.removeAll(view.getCrashedMembers()); viewReplyProcessor.initialize(id, responders); viewReplyProcessor.processPendingRequests(pendingLeaves, pendingRemovals);
@Test public void testRemoveMembers() throws Exception { int numMembers = members.size(); NetView view = new NetView(members.get(0), 2, new ArrayList<>(members)); setFailureDetectionPorts(view); for (int i = 1; i < numMembers; i += 2) { view.remove(members.get(i)); assertFalse(view.contains(members.get(i))); } List<InternalDistributedMember> remainingMembers = view.getMembers(); int num = remainingMembers.size(); for (int i = 0; i < num; i++) { InternalDistributedMember mbr = remainingMembers.get(i); assertEquals(mbr.getPort(), view.getFailureDetectionPort(mbr)); } }
@Test public void testNullPublicKeysNotRetained() throws Exception { NetView view = new NetView(members.get(0), 2, new ArrayList<>(members)); setFailureDetectionPorts(view); NetView newView = new NetView(view, 3); for (InternalDistributedMember member : view.getMembers()) { view.setPublicKey(member, null); } newView.setPublicKeys(view); for (InternalDistributedMember member : view.getMembers()) { assertNull(newView.getPublicKey(member)); assertNull(view.getPublicKey(member)); } }