public static void finishRebalance(CacheTopology.Phase nextPhase, BlockingLocalTopologyManager... topologyManagers) throws InterruptedException { switch (nextPhase) { case READ_OLD_WRITE_ALL: confirmTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL, topologyManagers); // fallthrough case READ_ALL_WRITE_ALL: confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL, topologyManagers); // fallthrough case READ_NEW_WRITE_ALL: confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL, topologyManagers); // fallthrough case NO_REBALANCE: confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE, topologyManagers); } }
public static void confirmTopologyUpdate(CacheTopology.Phase phase, BlockingLocalTopologyManager... topologyManagers) throws InterruptedException { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectTopologyUpdate(phase).unblock(); } if (needConfirmation(phase)) { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectPhaseConfirmation().unblock(); } } }
void fail(Throwable e) { if (latch.isCompletedExceptionally()) { latch.join(); } log.errorf(e, "Failed waiting for test to unblock %s %d on %s", type, topologyId, address); failManager(e); latch.completeExceptionally(e); }
BlockingLocalTopologyManager.replaceTopologyManagerDefaultCache(cacheManagers.get(0)); BlockingLocalTopologyManager localTopologyManager2 = BlockingLocalTopologyManager.replaceTopologyManagerDefaultCache(cacheManagers.get(2)); confirmTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL, localTopologyManager0, localTopologyManager2); confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL, localTopologyManager0, localTopologyManager2); localTopologyManager0.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); localTopologyManager2.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); BlockingLocalTopologyManager.BlockedConfirmation blockedConfirmation0 = localTopologyManager0.expectPhaseConfirmation(); BlockingLocalTopologyManager.BlockedConfirmation blockedConfirmation2 = localTopologyManager2.expectPhaseConfirmation(); BlockingLocalTopologyManager.BlockedTopology blockedTopology0 = localTopologyManager0.expectTopologyUpdate(); BlockingLocalTopologyManager.BlockedTopology blockedTopology2 = localTopologyManager2.expectTopologyUpdate(); localTopologyManager0.expectPhaseConfirmation().unblock(); localTopologyManager2.expectPhaseConfirmation().unblock(); confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL, localTopologyManager0, localTopologyManager2); confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL, localTopologyManager0, localTopologyManager2); confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE, localTopologyManager0, localTopologyManager2); localTopologyManager2.stopBlocking(); localTopologyManager0.stopBlocking();
confirmTopologyUpdate(Phase.READ_OLD_WRITE_ALL, topologyManager0, topologyManager1, joiner.topologyManager); topologyManager0.confirmTopologyUpdate(Phase.READ_ALL_WRITE_ALL); awaitForTopology(currentTopologyId + 2, cache(0)); topologyManager1.confirmTopologyUpdate(Phase.READ_ALL_WRITE_ALL); joiner.topologyManager.confirmTopologyUpdate(Phase.READ_ALL_WRITE_ALL); topologyManager1.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); awaitForTopology(currentTopologyId + 3, cache(1)); joiner.topologyManager.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); topologyManager0.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); if (retryTopologyOnNode2 < 3) { joiner.topologyManager.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); finishRebalance(Phase.NO_REBALANCE, topologyManager0, topologyManager1, joiner.topologyManager);
private void doTest(Operation operation, boolean init) throws Exception { final MagicKey key = new MagicKey(cache(1), cache(2)); if (init) { cache(0).put(key, "v1"); assertInAllCache(key, "v1"); } BlockingLocalTopologyManager bltm0 = replaceTopologyManagerDefaultCache(manager(0)); BlockingLocalTopologyManager bltm1 = replaceTopologyManagerDefaultCache(manager(1)); killMember(2, null, false); //CH_UPDATE + REBALANCE_START + CH_UPDATE(blocked) bltm0.expectRebalanceStartAfterLeave().unblock(); bltm1.expectRebalanceStartAfterLeave().unblock(); bltm0.expectPhaseConfirmation().unblock(); bltm1.expectPhaseConfirmation().unblock(); //check if we are in the correct state LocalizedCacheTopology cacheTopology = TestingUtil.extractComponent(cache(0), DistributionManager.class).getCacheTopology(); DistributionInfo distributionInfo = cacheTopology.getDistribution(key); assertFalse(distributionInfo.isReadOwner()); assertTrue(distributionInfo.isWriteOwner()); assertEquals(address(1), distributionInfo.primary()); operation.put(key, "v2", cache(1)); BlockingLocalTopologyManager.finishRebalance(CacheTopology.Phase.READ_ALL_WRITE_ALL, bltm0, bltm1); waitForClusterToForm(); //let the cluster finish the state transfer assertInAllCache(key, "v2"); }
private BlockingLocalTopologyManager.BlockedTopology blockNewRebalance(BlockingLocalTopologyManager ltm, BlockingLocalTopologyManager.BlockedTopology blockedTopology) throws InterruptedException { if (blockedTopology.getCacheTopology().getPhase() == CacheTopology.Phase.READ_NEW_WRITE_ALL) { // Block the rebalance start first, and only then unblock the previous topology // Otherwise the order of the rebalance start and topology confirmation wouldn't be deterministic BlockingLocalTopologyManager.BlockedTopology newTopology = ltm.expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); blockedTopology.unblock(); ltm.expectPhaseConfirmation().unblock(); return newTopology; } else { assertEquals(CacheTopology.Phase.READ_OLD_WRITE_ALL, blockedTopology.getCacheTopology().getPhase()); ltm.confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL); return blockedTopology; } }
BlockingLocalTopologyManager ltm2 = BlockingLocalTopologyManager.replaceTopologyManager(manager(2), cacheName); ControlledTransport transport0 = ControlledTransport.replace(cache(0)); ControlledTransport transport1 = ControlledTransport.replace(cache(1)); BlockedTopology t1 = ltm2.expectTopologyUpdate(CacheTopology.Phase.NO_REBALANCE, stableTopologyId + 1); if (t1.getCacheTopology().getTopologyId() == stableTopologyId + 1) assertEquals(CacheTopology.Phase.NO_REBALANCE, t1.getPhase()); BlockedTopology t2 = ltm2.expectTopologyUpdate(CacheTopology.Phase.TRANSITORY, stableTopologyId + 2); assertEquals(CacheTopology.Phase.TRANSITORY, t2.getPhase()); assertEquals(2, t2.getCacheTopology().getActualMembers().size()); BlockedTopology t3 = ltm2.expectTopologyUpdate(CacheTopology.Phase.NO_REBALANCE, stableTopologyId + 3); assertEquals(2, t3.getCacheTopology().getActualMembers().size()); assertEquals(null, t3.getCacheTopology().getPendingCH()); ltm2.stopBlocking(); t3.unblock();
BlockingLocalTopologyManager.replaceTopologyManagerDefaultCache(testCaches.controllerCache.getCacheManager()); final CheckPoint checkPoint = new CheckPoint(); topologyManager.confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE); topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL); topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL); topologyManager.confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE);
int startTopologyId = c1.getAdvancedCache().getDistributionManager().getCacheTopology().getTopologyId(); BlockingLocalTopologyManager bltm = BlockingLocalTopologyManager.replaceTopologyManager(cm4, CACHE_NAME); bltm.confirmTopologyUpdate(CacheTopology.Phase.TRANSITORY); assertEquals(0, stateAppliedLatch.getCount()); bltm.confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE);
confirmTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL, nodeAController.topologyManager, nodeBController.topologyManager, nodeBController.topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL); assertEquals(0, nodeC.commandLatch.getCount()); nodeAController.topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL); nodeC.controller.topologyManager.confirmTopologyUpdate(CacheTopology.Phase.READ_ALL_WRITE_ALL); confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL, nodeAController.topologyManager, nodeBController.topologyManager, nodeC.controller.topologyManager); confirmTopologyUpdate(CacheTopology.Phase.NO_REBALANCE, nodeAController.topologyManager, nodeBController.topologyManager, nodeC.controller.topologyManager); awaitForTopology(currentTopologyId + 4, cache(0)); assertNull("Wrong put() return value.", tx.get()); nodeAController.topologyManager.stopBlocking(); nodeBController.topologyManager.stopBlocking(); nodeC.controller.topologyManager.stopBlocking();
/** * x-site state transfer is triggered during a cache topology change. */ private void doXSiteStateTransferDuringTopologyChange(TopologyEvent event) throws Exception { log.debugf("Start topology change during x-site state transfer with %s", event); initBeforeTest(); final TestCaches<Object, Object> testCaches = createTestCache(event, LON); log.debugf("Controlled cache=%s, Coordinator cache=%s, Cache to remove=%s", addressOf(testCaches.controllerCache), addressOf(testCaches.coordinator), testCaches.removeIndex < 0 ? "NONE" : addressOf(cache(LON, testCaches.removeIndex))); BlockingLocalTopologyManager topologyManager = replaceTopologyManagerDefaultCache(testCaches.controllerCache.getCacheManager()); final Future<Void> topologyEventFuture = triggerTopologyChange(LON, testCaches.removeIndex); // We could get either the NO_REBALANCE update or the READ_OLD rebalance start first BlockingLocalTopologyManager.BlockedTopology blockedTopology = topologyManager.expectTopologyUpdate(); log.debug("Start x-site state transfer"); startStateTransfer(testCaches.coordinator, NYC); assertOnline(LON, NYC); blockedTopology.unblock(); topologyEventFuture.get(); awaitLocalStateTransfer(LON); awaitXSiteStateSent(LON); assertData(); }
private BlockingLocalTopologyManager replaceTopologyManager(EmbeddedCacheManager cacheContainer) { BlockingLocalTopologyManager localTopologyManager = BlockingLocalTopologyManager.replaceTopologyManagerDefaultCache(cacheContainer); topologyManagerList.add(localTopologyManager); return localTopologyManager; }
public BlockedTopology expectRebalanceStartAfterLeave() throws InterruptedException { // After a node leaves, the coordinator sends a NO_REBALANCE topology update with the remaining members // Then it immediately starts the rebalance with a READ_OLD topology update // The members can receive the topology updates in either order BlockedTopology topology0 = expectTopologyUpdate(); if (topology0.getType() == Type.REBALANCE_START) { expectTopologyUpdate(CacheTopology.Phase.NO_REBALANCE).unblock(); } else { topology0.unblock(); topology0 = expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); } return topology0; }
public static BlockingLocalTopologyManager replaceTopologyManagerDefaultCache(EmbeddedCacheManager cacheContainer) { return replaceTopologyManager(cacheContainer, CacheContainer.DEFAULT_CACHE_NAME); }
public BlockedConfirmation expectPhaseConfirmation(int topologyId) throws InterruptedException { BlockedConfirmation blockedConfirmation = expectPhaseConfirmation(); assertEquals(topologyId, blockedConfirmation.getTopologyId()); return blockedConfirmation; }
public static BlockingLocalTopologyManager replaceTopologyManager(EmbeddedCacheManager cacheContainer, String cacheName) { LocalTopologyManager manager = TestingUtil.extractGlobalComponent(cacheContainer, LocalTopologyManager.class); BlockingLocalTopologyManager controlledLocalTopologyManager = new BlockingLocalTopologyManager(manager, cacheContainer.getAddress(), cacheName); TestingUtil.replaceComponent(cacheContainer, LocalTopologyManager.class, controlledLocalTopologyManager, true); return controlledLocalTopologyManager; }
new WaitForTopologyInterceptor(currentTopologyId + topologyOnNode2))); confirmTopologyUpdate(Phase.READ_OLD_WRITE_ALL, topologyManager0, topologyManager1, joiner.topologyManager); confirmTopologyUpdate(Phase.READ_ALL_WRITE_ALL, topologyManager0, topologyManager1, joiner.topologyManager); topologyManager0.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); joiner.topologyManager.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); topologyManager1.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); topologyManager0.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); joiner.topologyManager.confirmTopologyUpdate(Phase.READ_NEW_WRITE_ALL); finishRebalance(Phase.NO_REBALANCE, topologyManager0, topologyManager1, joiner.topologyManager);
private BlockingLocalTopologyManager replaceTopologyManager(EmbeddedCacheManager cacheContainer) { BlockingLocalTopologyManager localTopologyManager = BlockingLocalTopologyManager.replaceTopologyManagerDefaultCache(cacheContainer); topologyManagerList.add(localTopologyManager); return localTopologyManager; }
public BlockedTopology expectTopologyUpdate(CacheTopology.Phase phase) throws InterruptedException { BlockedTopology blockedTopology = expectTopologyUpdate(); assertTrue("Expected a CH_UPDATE or REBALANCE_START, but got a CONFIRMATION", blockedTopology.getType() != Type.CONFIRMATION); assertEquals(phase, blockedTopology.getCacheTopology().getPhase()); return blockedTopology; }