X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-akka-raft%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fraft%2Fbehaviors%2FPartitionedLeadersElectionScenarioTest.java;fp=opendaylight%2Fmd-sal%2Fsal-akka-raft%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fraft%2Fbehaviors%2FPartitionedLeadersElectionScenarioTest.java;h=9ad526cadbc413e09b93195e5a4d48df4e93d3fa;hp=0000000000000000000000000000000000000000;hb=5996901683ffd1a5710a5507f4c0e3b377d34802;hpb=ab6ec7799915fefe5a1d7144f4b10d3aef697fc9 diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java new file mode 100644 index 0000000000..9ad526cadb --- /dev/null +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.raft.behaviors; + +import static org.junit.Assert.assertEquals; +import akka.actor.ActorRef; +import com.google.common.collect.ImmutableMap; +import org.junit.Test; +import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl; +import org.opendaylight.controller.cluster.raft.RaftState; +import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.messages.AppendEntries; +import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply; +import org.opendaylight.controller.cluster.raft.messages.RequestVote; +import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply; + +/** + * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members. + * + * @author Thomas Pantelis + */ +public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest { + + /** + * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3 + * sends a heartbeat first when connectivity is re-established. + */ + @Test + public void runTest1() throws Exception { + testLog.info("PartitionedLeadersElectionScenarioTest 1 starting"); + + setupInitialMemberBehaviors(); + + sendInitialElectionTimeoutToFollowerMember2(); + + sendInitialElectionTimeoutToFollowerMember3(); + + sendElectionTimeoutToNowCandidateMember2(); + + resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst(); + + testLog.info("PartitionedLeadersElectionScenarioTest 1 ending"); + } + + /** + * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2 + * sends a heartbeat first when connectivity is re-established. + */ + @Test + public void runTest2() throws Exception { + testLog.info("PartitionedLeadersElectionScenarioTest 2 starting"); + + setupInitialMemberBehaviors(); + + sendInitialElectionTimeoutToFollowerMember2(); + + sendInitialElectionTimeoutToFollowerMember3(); + + sendElectionTimeoutToNowCandidateMember2(); + + resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst(); + + testLog.info("PartitionedLeadersElectionScenarioTest 2 ending"); + } + + private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() { + testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting"); + + // Re-establish connectivity between member 2 and 3, ie stop dropping messages between + // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should + // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to + // Follower as its term is less than member 2's. + + member1Actor.clear(); + member1Actor.expectMessageClass(AppendEntries.class, 1); + + member2Actor.clear(); + member2Actor.expectMessageClass(AppendEntriesReply.class, 1); + + member3Actor.clear(); + member3Actor.expectMessageClass(AppendEntries.class, 1); + + sendHeartbeat(member2ActorRef); + + member1Actor.waitForExpectedMessages(AppendEntries.class); + member3Actor.waitForExpectedMessages(AppendEntries.class); + + member2Actor.waitForExpectedMessages(AppendEntriesReply.class); + + verifyBehaviorState("member 1", member1Actor, RaftState.Follower); + verifyBehaviorState("member 2", member2Actor, RaftState.Leader); + verifyBehaviorState("member 3", member3Actor, RaftState.Follower); + + assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm()); + assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm()); + assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm()); + + testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending"); + } + + private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception { + testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting"); + + // Re-establish connectivity between member 2 and 3, ie stop dropping messages between + // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send + // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2). + // This should cause member 3 to switch to Follower. + + member1Actor.clear(); + member1Actor.expectMessageClass(AppendEntries.class, 1); + + member2Actor.clear(); + member2Actor.expectMessageClass(AppendEntries.class, 1); + + member3Actor.clear(); + member3Actor.expectMessageClass(AppendEntriesReply.class, 1); + + sendHeartbeat(member3ActorRef); + + member3Actor.waitForExpectedMessages(AppendEntriesReply.class); + + AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class); + assertEquals("isSuccess", false, appendEntriesReply.isSuccess()); + assertEquals("getTerm", 3, appendEntriesReply.getTerm()); + + verifyBehaviorState("member 1", member1Actor, RaftState.Follower); + verifyBehaviorState("member 2", member2Actor, RaftState.Leader); + verifyBehaviorState("member 3", member3Actor, RaftState.Follower); + + assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm()); + assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm()); + assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm()); + + testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending"); + } + + private void sendElectionTimeoutToNowCandidateMember2() throws Exception { + testLog.info("sendElectionTimeoutToNowCandidateMember2 starting"); + + // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any + // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply + // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3 + // are dropped to simulate loss of network connectivity. Note member 2 will increment its + // election term to 3. + + member1Actor.clear(); + member1Actor.expectMessageClass(AppendEntries.class, 1); + + member2Actor.clear(); + member2Actor.expectMessageClass(RequestVoteReply.class, 1); + member2Actor.expectMessageClass(AppendEntriesReply.class, 1); + + member3Actor.clear(); + member3Actor.dropMessagesToBehavior(AppendEntries.class); + member3Actor.dropMessagesToBehavior(RequestVote.class); + + member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender()); + + member2Actor.waitForExpectedMessages(RequestVoteReply.class); + + RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class); + assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm()); + assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted()); + + member3Actor.waitForExpectedMessages(RequestVote.class); + + member1Actor.waitForExpectedMessages(AppendEntries.class); + member3Actor.waitForExpectedMessages(AppendEntries.class); + member2Actor.waitForExpectedMessages(AppendEntriesReply.class); + + // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3 + // is 3 and member 3's term is 2. + + verifyBehaviorState("member 1", member1Actor, RaftState.Follower); + verifyBehaviorState("member 2", member2Actor, RaftState.Leader); + verifyBehaviorState("member 3", member3Actor, RaftState.Leader); + + assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm()); + assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm()); + assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm()); + + testLog.info("sendElectionTimeoutToNowCandidateMember2 ending"); + } + + private void sendInitialElectionTimeoutToFollowerMember3() throws Exception { + testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting"); + + // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1). + // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower, + // should reply and grant the vote but member 2 will drop the message to simulate loss of network + // connectivity between members 2 and 3. member 3 should switch to leader. + + member1Actor.clear(); + member1Actor.expectMessageClass(RequestVote.class, 1); + member1Actor.expectMessageClass(AppendEntries.class, 1); + + member2Actor.clear(); + member2Actor.dropMessagesToBehavior(RequestVote.class); + member2Actor.dropMessagesToBehavior(AppendEntries.class); + + member3Actor.clear(); + member3Actor.expectMessageClass(RequestVoteReply.class, 1); + member3Actor.expectMessageClass(AppendEntriesReply.class, 1); + + member3ActorRef.tell(new ElectionTimeout(), ActorRef.noSender()); + + member1Actor.waitForExpectedMessages(RequestVote.class); + member2Actor.waitForExpectedMessages(RequestVote.class); + member3Actor.waitForExpectedMessages(RequestVoteReply.class); + + RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class); + assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm()); + assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted()); + + // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to + // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The + // AppendEntries message to member 2 is dropped. + + member1Actor.waitForExpectedMessages(AppendEntries.class); + member2Actor.waitForExpectedMessages(AppendEntries.class); + member3Actor.waitForExpectedMessages(AppendEntriesReply.class); + + verifyBehaviorState("member 1", member1Actor, RaftState.Follower); + verifyBehaviorState("member 2", member2Actor, RaftState.Candidate); + verifyBehaviorState("member 3", member3Actor, RaftState.Leader); + + assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm()); + assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm()); + assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm()); + + testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending"); + } + + private void sendInitialElectionTimeoutToFollowerMember2() { + testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting"); + + // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1). + // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages. + // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it + // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to + // simulate loss of network connectivity between members 2 and 3. + + member1Actor.expectMessageClass(RequestVote.class, 1); + + member2Actor.expectBehaviorStateChange(); + + member3Actor.dropMessagesToBehavior(RequestVote.class); + + member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender()); + + member1Actor.waitForExpectedMessages(RequestVote.class); + member3Actor.waitForExpectedMessages(RequestVote.class); + + // Original leader member 1 should switch to Follower as the RequestVote term is greater than its + // term. It won't send back a RequestVoteReply in this case. + + verifyBehaviorState("member 1", member1Actor, RaftState.Follower); + + // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members. + + member2Actor.waitForBehaviorStateChange(); + verifyBehaviorState("member 2", member2Actor, RaftState.Candidate); + + assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm()); + assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm()); + assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm()); + + testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending"); + } + + private void setupInitialMemberBehaviors() throws Exception { + testLog.info("setupInitialMemberBehaviors starting"); + + // Create member 2's behavior initially as Follower + + member2Context = newRaftActorContext("member2", member2ActorRef, + ImmutableMap.builder(). + put("member1", member1ActorRef.path().toString()). + put("member3", member3ActorRef.path().toString()).build()); + + DefaultConfigParamsImpl member2ConfigParams = newConfigParams(); + member2Context.setConfigParams(member2ConfigParams); + + Follower member2Behavior = new Follower(member2Context); + member2Actor.behavior = member2Behavior; + + // Create member 3's behavior initially as Follower + + member3Context = newRaftActorContext("member3", member3ActorRef, + ImmutableMap.builder(). + put("member1", member1ActorRef.path().toString()). + put("member2", member2ActorRef.path().toString()).build()); + + DefaultConfigParamsImpl member3ConfigParams = newConfigParams(); + member3Context.setConfigParams(member3ConfigParams); + + Follower member3Behavior = new Follower(member3Context); + member3Actor.behavior = member3Behavior; + + // Create member 1's behavior initially as Leader + + member1Context = newRaftActorContext("member1", member1ActorRef, + ImmutableMap.builder(). + put("member2", member2ActorRef.path().toString()). + put("member3", member3ActorRef.path().toString()).build()); + + DefaultConfigParamsImpl member1ConfigParams = newConfigParams(); + member1Context.setConfigParams(member1ConfigParams); + + initializeLeaderBehavior(member1Actor, member1Context, 2); + + member2Actor.clear(); + member3Actor.clear(); + + testLog.info("setupInitialMemberBehaviors ending"); + } +}