public BlockedTopology expectTopologyUpdate() throws InterruptedException { Event update = queuedTopologies.poll(TIMEOUT_SECONDS, TimeUnit.SECONDS); if (update == null) { throw new TimeoutException("Timed out waiting for topology update on " + address); } return new BlockedTopology(update); }
private void assertOwners(BlockedTopology t, boolean current, int segmentId, Address... address) { ConsistentHash ch = current ? t.getCacheTopology().getCurrentCH() : t.getCacheTopology().getPendingCH(); assertEquals("Topology: " + t.getCacheTopology(), Arrays.asList(address), ch.locateOwnersForSegment(segmentId)); } }
localTopologyManager0.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); localTopologyManager2.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); BlockingLocalTopologyManager.BlockedConfirmation blockedConfirmation0 = localTopologyManager0.expectPhaseConfirmation(); blockedTopology0.unblock(); blockedTopology2.unblock(); localTopologyManager0.expectPhaseConfirmation().unblock(); localTopologyManager2.expectPhaseConfirmation().unblock();
if (t1.getCacheTopology().getTopologyId() == stableTopologyId + 1) assertEquals(CacheTopology.Phase.NO_REBALANCE, t1.getPhase()); assertEquals(2, t1.getCacheTopology().getActualMembers().size()); assertEquals(null, t1.getCacheTopology().getPendingCH()); assertOwners(t1, true, 0); assertOwners(t1, true, 1, address(1)); assertOwners(t1, true, 2, address(2)); t1.unblock(); assertEquals(CacheTopology.Phase.TRANSITORY, t2.getPhase()); assertEquals(2, t2.getCacheTopology().getActualMembers().size()); assertNotNull(t2.getCacheTopology().getPendingCH()); assertOwners(t2, false, 0, address(2)); assertOwners(t2, false, 1, address(1)); assertOwners(t2, false, 2, address(2)); t2.unblock(); assertEquals(2, t3.getCacheTopology().getActualMembers().size()); assertEquals(null, t3.getCacheTopology().getPendingCH()); t3.unblock(); if (t3.getCacheTopology().getCurrentCH().locatePrimaryOwnerForSegment(0) == null) { ControlledRpcManager.BlockedRequest cancelStateTransfer = rpcManager2.expectCommand(StateRequestCommand.class, request -> assertEquals(StateRequestCommand.Type.CANCEL_STATE_TRANSFER, request.getType()));
private void doTest(Operation operation, boolean init) throws Exception { final MagicKey key = new MagicKey(cache(1), cache(2)); if (init) { cache(0).put(key, "v1"); assertInAllCache(key, "v1"); } BlockingLocalTopologyManager bltm0 = replaceTopologyManagerDefaultCache(manager(0)); BlockingLocalTopologyManager bltm1 = replaceTopologyManagerDefaultCache(manager(1)); killMember(2, null, false); //CH_UPDATE + REBALANCE_START + CH_UPDATE(blocked) bltm0.expectRebalanceStartAfterLeave().unblock(); bltm1.expectRebalanceStartAfterLeave().unblock(); bltm0.expectPhaseConfirmation().unblock(); bltm1.expectPhaseConfirmation().unblock(); //check if we are in the correct state LocalizedCacheTopology cacheTopology = TestingUtil.extractComponent(cache(0), DistributionManager.class).getCacheTopology(); DistributionInfo distributionInfo = cacheTopology.getDistribution(key); assertFalse(distributionInfo.isReadOwner()); assertTrue(distributionInfo.isWriteOwner()); assertEquals(address(1), distributionInfo.primary()); operation.put(key, "v2", cache(1)); BlockingLocalTopologyManager.finishRebalance(CacheTopology.Phase.READ_ALL_WRITE_ALL, bltm0, bltm1); waitForClusterToForm(); //let the cluster finish the state transfer assertInAllCache(key, "v2"); }
/** * x-site state transfer is triggered during a cache topology change. */ private void doXSiteStateTransferDuringTopologyChange(TopologyEvent event) throws Exception { log.debugf("Start topology change during x-site state transfer with %s", event); initBeforeTest(); final TestCaches<Object, Object> testCaches = createTestCache(event, LON); log.debugf("Controlled cache=%s, Coordinator cache=%s, Cache to remove=%s", addressOf(testCaches.controllerCache), addressOf(testCaches.coordinator), testCaches.removeIndex < 0 ? "NONE" : addressOf(cache(LON, testCaches.removeIndex))); BlockingLocalTopologyManager topologyManager = replaceTopologyManagerDefaultCache(testCaches.controllerCache.getCacheManager()); final Future<Void> topologyEventFuture = triggerTopologyChange(LON, testCaches.removeIndex); // We could get either the NO_REBALANCE update or the READ_OLD rebalance start first BlockingLocalTopologyManager.BlockedTopology blockedTopology = topologyManager.expectTopologyUpdate(); log.debug("Start x-site state transfer"); startStateTransfer(testCaches.coordinator, NYC); assertOnline(LON, NYC); blockedTopology.unblock(); topologyEventFuture.get(); awaitLocalStateTransfer(LON); awaitXSiteStateSent(LON); assertData(); }
private BlockingLocalTopologyManager.BlockedTopology blockNewRebalance(BlockingLocalTopologyManager ltm, BlockingLocalTopologyManager.BlockedTopology blockedTopology) throws InterruptedException { if (blockedTopology.getCacheTopology().getPhase() == CacheTopology.Phase.READ_NEW_WRITE_ALL) { // Block the rebalance start first, and only then unblock the previous topology // Otherwise the order of the rebalance start and topology confirmation wouldn't be deterministic BlockingLocalTopologyManager.BlockedTopology newTopology = ltm.expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); blockedTopology.unblock(); ltm.expectPhaseConfirmation().unblock(); return newTopology; } else { assertEquals(CacheTopology.Phase.READ_OLD_WRITE_ALL, blockedTopology.getCacheTopology().getPhase()); ltm.confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL); return blockedTopology; } }
public BlockedTopology expectRebalanceStartAfterLeave() throws InterruptedException { // After a node leaves, the coordinator sends a NO_REBALANCE topology update with the remaining members // Then it immediately starts the rebalance with a READ_OLD topology update // The members can receive the topology updates in either order BlockedTopology topology0 = expectTopologyUpdate(); if (topology0.getType() == Type.REBALANCE_START) { expectTopologyUpdate(CacheTopology.Phase.NO_REBALANCE).unblock(); } else { topology0.unblock(); topology0 = expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); } return topology0; }
public static void confirmTopologyUpdate(CacheTopology.Phase phase, BlockingLocalTopologyManager... topologyManagers) throws InterruptedException { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectTopologyUpdate(phase).unblock(); } if (needConfirmation(phase)) { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectPhaseConfirmation().unblock(); } } }
/** * Expect a topology updates and unblock it. * <p> * If the update requires confirmation, unblock the confirmation as well. */ public void confirmTopologyUpdate(CacheTopology.Phase phase) throws InterruptedException { expectTopologyUpdate(phase).unblock(); if (needConfirmation(phase)) { expectPhaseConfirmation().unblock(); } }
public BlockedTopology expectTopologyUpdate(CacheTopology.Phase phase, int topologyId) throws InterruptedException { BlockedTopology blockedTopology = expectTopologyUpdate(); assertEquals(topologyId, blockedTopology.getCacheTopology().getTopologyId()); assertEquals(phase, blockedTopology.getCacheTopology().getPhase()); return blockedTopology; }
public BlockedTopology expectTopologyUpdate(CacheTopology.Phase phase) throws InterruptedException { BlockedTopology blockedTopology = expectTopologyUpdate(); assertTrue("Expected a CH_UPDATE or REBALANCE_START, but got a CONFIRMATION", blockedTopology.getType() != Type.CONFIRMATION); assertEquals(phase, blockedTopology.getCacheTopology().getPhase()); return blockedTopology; }