private Stream<RaftServerImpl> getServerStream(RaftGroupId groupId) { final Stream<RaftServerProxy> stream = getRaftServerProxyStream(groupId); return groupId != null? stream.filter(s -> s.containsGroup(groupId)).map(s -> RaftServerTestUtil.getRaftServerImpl(s, groupId)) : stream.flatMap(s -> RaftServerTestUtil.getRaftServerImpls(s).stream()); }
public void assertServer(MiniRaftCluster cluster, ClientId clientId, long callId, long oldLastApplied) throws Exception { long leaderApplied = cluster.getLeader().getState().getLastAppliedIndex(); // make sure retry cache has the entry for (RaftServerImpl server : cluster.iterateServerImpls()) { LOG.info("check server " + server.getId()); if (server.getState().getLastAppliedIndex() < leaderApplied) { Thread.sleep(1000); } Assert.assertEquals(2, RaftServerTestUtil.getRetryCacheSize(server)); Assert.assertNotNull(RaftServerTestUtil.getRetryEntry(server, clientId, callId)); // make sure there is only one log entry committed Assert.assertEquals(1, count(server.getState().getLog(), oldLastApplied + 1)); } }
public static void waitAndCheckNewConf(MiniRaftCluster cluster, RaftPeer[] peers, int numOfRemovedPeers, Collection<String> deadPeers) throws Exception { final long sleepMs = cluster.getMaxTimeout() * (numOfRemovedPeers + 2); JavaUtils.attempt(() -> waitAndCheckNewConf(cluster, peers, deadPeers), 3, sleepMs, "waitAndCheckNewConf", LOG); } private static void waitAndCheckNewConf(MiniRaftCluster cluster,
Objects.requireNonNull(reply.getStateMachineException()); RetryCache.CacheEntry oldEntry = RaftServerTestUtil.getRetryEntry( cluster.getLeader(), client.getId(), callId); Assert.assertNotNull(oldEntry); Assert.assertTrue(RaftServerTestUtil.isRetryCacheEntryFailed(oldEntry)); Objects.requireNonNull(reply.getStateMachineException()); RetryCache.CacheEntry currentEntry = RaftServerTestUtil.getRetryEntry( cluster.getLeader(), client.getId(), callId); Assert.assertNotNull(currentEntry); Assert.assertTrue(RaftServerTestUtil.isRetryCacheEntryFailed(currentEntry)); Assert.assertNotEquals(oldEntry, currentEntry);
public RaftServerImpl getRaftServerImpl(RaftServerProxy proxy) { return RaftServerTestUtil.getRaftServerImpl(proxy, getGroupId()); }
RaftServerTestUtil.getRetryEntry(server, client.getId(), callId)); Assert.assertEquals(oldLastApplied + 1, server.getState().getLastAppliedIndex());
RaftServerTestUtil.getLogAppenders(cluster.getLeader()).forEach(logAppender -> JavaUtils.runAsUnchecked(() -> JavaUtils.attempt(() -> { final long leaderNextIndex = leaderLog.getNextIndex();
Objects.requireNonNull(reply.getStateMachineException()); RetryCache.CacheEntry oldEntry = RaftServerTestUtil.getRetryEntry( leader, client.getId(), callId); Assert.assertNotNull(oldEntry); Assert.assertTrue(RaftServerTestUtil.isRetryCacheEntryFailed(oldEntry)); Objects.requireNonNull(reply.getStateMachineException()); RetryCache.CacheEntry currentEntry = RaftServerTestUtil.getRetryEntry( leader, client.getId(), callId); Assert.assertNotNull(currentEntry); Assert.assertTrue(RaftServerTestUtil.isRetryCacheEntryFailed(currentEntry)); Assert.assertNotEquals(oldEntry, currentEntry); failPreAppend = false;
@Test public void testStateMachineRegistry() throws Throwable { final Map<RaftGroupId, StateMachine> registry = new ConcurrentHashMap<>(); registry.put(RaftGroupId.randomId(), new SimpleStateMachine4Testing()); registry.put(RaftGroupId.randomId(), new SMTransactionContext()); try(MiniRaftClusterWithSimulatedRpc cluster = newCluster(0)) { cluster.setStateMachineRegistry(registry::get); final RaftPeerId id = RaftPeerId.valueOf("s0"); cluster.putNewServer(id, null, true); cluster.start(); for(RaftGroupId gid : registry.keySet()) { final RaftGroup newGroup = RaftGroup.valueOf(gid, cluster.getPeers()); LOG.info("add new group: " + newGroup); final RaftClient client = cluster.createClient(newGroup); for(RaftPeer p : newGroup.getPeers()) { client.groupAdd(newGroup, p.getId()); } } final RaftServerProxy proxy = cluster.getServer(id); for(Map.Entry<RaftGroupId, StateMachine> e: registry.entrySet()) { final RaftServerImpl impl = RaftServerTestUtil.getRaftServerImpl(proxy, e.getKey()); Assert.assertSame(e.getValue(), impl.getStateMachine()); } } } }
RaftServerTestUtil.getRetryEntry(server, client.getId(), callId)); final RaftLog log = server.getState().getLog(); RaftTestUtil.logEntriesContains(log, oldLastApplied + 1, log.getNextIndex(), message);
RaftServerTestUtil.getLogAppenders(cluster.getLeader()).forEach(logAppender -> {
Thread.sleep(1000); Assert.assertEquals(2, RaftServerTestUtil.getRetryCacheSize(server)); Assert.assertNotNull( RaftServerTestUtil.getRetryEntry(server, client.getId(), callId));
private void runTestAddRemovePeers(boolean leaderStepdown, CLUSTER cluster) throws Exception { RaftTestUtil.waitForLeader(cluster); PeerChanges change = cluster.addNewPeers(2, true); RaftPeer[] allPeers = cluster.removePeers(2, leaderStepdown, asList(change.newPeers)).allPeersInNewConf; // trigger setConfiguration cluster.setConfiguration(allPeers); // wait for the new configuration to take effect waitAndCheckNewConf(cluster, allPeers, 2, null); }
final RaftServerImpl impl = RaftServerTestUtil.getRaftServerImpl(server, cluster.getGroupId()); if (impl.isAlive()) { JavaUtils.attempt(() -> RaftTestUtil.assertLogEntries(impl, term, messages),
public static void waitAndCheckNewConf(MiniRaftCluster cluster, RaftPeer[] peers, int numOfRemovedPeers, Collection<RaftPeerId> deadPeers) throws Exception { final TimeDuration sleepTime = cluster.getTimeoutMax().apply(n -> n * (numOfRemovedPeers + 2)); JavaUtils.attempt(() -> waitAndCheckNewConf(cluster, peers, deadPeers), 10, sleepTime, "waitAndCheckNewConf", LOG); } private static void waitAndCheckNewConf(MiniRaftCluster cluster,
/** * remove 2 peers (5 peers -> 3 peers), no leader change */ @Test public void testRemovePeers() throws Exception { runWithNewCluster(5, cluster -> { RaftTestUtil.waitForLeader(cluster); // remove peers, leader still included in the new conf RaftPeer[] allPeers = cluster .removePeers(2, false, Collections.emptyList()).allPeersInNewConf; // trigger setConfiguration cluster.setConfiguration(allPeers); // wait for the new configuration to take effect waitAndCheckNewConf(cluster, allPeers, 2, null); }); }
/** * remove 2 peers (5 peers -> 3 peers), no leader change */ @Test public void testRemovePeers() throws Exception { LOG.info("Start testRemovePeers"); MiniRaftCluster cluster = getCluster(5); cluster.start(); try { RaftTestUtil.waitForLeader(cluster); // remove peers, leader still included in the new conf RaftPeer[] allPeers = cluster .removePeers(2, false, Collections.emptyList()).allPeersInNewConf; // trigger setConfiguration cluster.setConfiguration(allPeers); // wait for the new configuration to take effect waitAndCheckNewConf(cluster, allPeers, 2, null); } finally { cluster.shutdown(); } }
/** * add 2 new peers (3 peers -> 5 peers), no leader change */ @Test public void testAddPeers() throws Exception { runWithNewCluster(3, cluster -> { RaftTestUtil.waitForLeader(cluster); // add new peers RaftPeer[] allPeers = cluster.addNewPeers(2, true).allPeersInNewConf; // trigger setConfiguration cluster.setConfiguration(allPeers); // wait for the new configuration to take effect waitAndCheckNewConf(cluster, allPeers, 0, null); }); }
Assert.assertTrue(reconf1.get()); Assert.assertTrue(reconf2.get()); waitAndCheckNewConf(cluster, finalPeers.get(), 2, null); final RaftPeerId leader2 = RaftTestUtil.waitForLeader(cluster).getId();
private void testAddRemovePeers(boolean leaderStepdown) throws Exception { MiniRaftCluster cluster = getCluster(5); cluster.start(); try { RaftTestUtil.waitForLeader(cluster); PeerChanges change = cluster.addNewPeers(2, true); RaftPeer[] allPeers = cluster.removePeers(2, leaderStepdown, asList(change.newPeers)).allPeersInNewConf; // trigger setConfiguration cluster.setConfiguration(allPeers); // wait for the new configuration to take effect waitAndCheckNewConf(cluster, allPeers, 2, null); } finally { cluster.shutdown(); } }