2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.raft.behaviors;
10 import static org.junit.Assert.assertEquals;
12 import akka.actor.ActorRef;
13 import com.google.common.collect.ImmutableMap;
14 import org.junit.Test;
15 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
16 import org.opendaylight.controller.cluster.raft.RaftState;
17 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
18 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
19 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
20 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
21 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
22 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
25 * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
27 * @author Thomas Pantelis
29 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
32 * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
33 * sends a heartbeat first when connectivity is re-established.
36 public void runTest1() throws Exception {
37 testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
39 setupInitialMemberBehaviors();
41 sendInitialElectionTimeoutToFollowerMember2();
43 sendInitialElectionTimeoutToFollowerMember3();
45 sendElectionTimeoutToNowCandidateMember2();
47 resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
49 testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
53 * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
54 * sends a heartbeat first when connectivity is re-established.
57 public void runTest2() throws Exception {
58 testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
60 setupInitialMemberBehaviors();
62 sendInitialElectionTimeoutToFollowerMember2();
64 sendInitialElectionTimeoutToFollowerMember3();
66 sendElectionTimeoutToNowCandidateMember2();
68 resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
70 testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
73 private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
74 testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
76 // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
77 // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
78 // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
79 // Follower as its term is less than member 2's.
82 member1Actor.expectMessageClass(AppendEntries.class, 1);
85 member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
88 member3Actor.expectMessageClass(AppendEntries.class, 1);
90 sendHeartbeat(member2ActorRef);
92 member1Actor.waitForExpectedMessages(AppendEntries.class);
93 member3Actor.waitForExpectedMessages(AppendEntries.class);
95 member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
97 verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
98 verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
99 verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
101 assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
102 assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
103 assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
105 testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
108 private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
109 testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
111 // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
112 // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
113 // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
114 // This should cause member 3 to switch to Follower.
116 member1Actor.clear();
117 member1Actor.expectMessageClass(AppendEntries.class, 1);
119 member2Actor.clear();
120 member2Actor.expectMessageClass(AppendEntries.class, 1);
122 member3Actor.clear();
123 member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
125 sendHeartbeat(member3ActorRef);
127 member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
129 AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
130 assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
131 assertEquals("getTerm", 3, appendEntriesReply.getTerm());
133 verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
134 verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
135 verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
137 assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
138 assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
139 assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
141 testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
144 private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
145 testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
147 // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
148 // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
149 // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
150 // are dropped to simulate loss of network connectivity. Note member 2 will increment its
151 // election term to 3.
153 member1Actor.clear();
154 member1Actor.expectMessageClass(AppendEntries.class, 1);
156 member2Actor.clear();
157 member2Actor.expectMessageClass(RequestVoteReply.class, 1);
158 member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
160 member3Actor.clear();
161 member3Actor.dropMessagesToBehavior(AppendEntries.class);
162 member3Actor.dropMessagesToBehavior(RequestVote.class);
164 member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
166 member2Actor.waitForExpectedMessages(RequestVoteReply.class);
168 RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
169 assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
170 assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
172 member3Actor.waitForExpectedMessages(RequestVote.class);
174 member1Actor.waitForExpectedMessages(AppendEntries.class);
175 member3Actor.waitForExpectedMessages(AppendEntries.class);
176 member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
178 // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
179 // is 3 and member 3's term is 2.
181 verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
182 verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
183 verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
185 assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
186 assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
187 assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
189 testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
192 private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
193 testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
195 // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
196 // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
197 // should reply and grant the vote but member 2 will drop the message to simulate loss of network
198 // connectivity between members 2 and 3. member 3 should switch to leader.
200 member1Actor.clear();
201 member1Actor.expectMessageClass(RequestVote.class, 1);
202 member1Actor.expectMessageClass(AppendEntries.class, 1);
204 member2Actor.clear();
205 member2Actor.dropMessagesToBehavior(RequestVote.class);
206 member2Actor.dropMessagesToBehavior(AppendEntries.class);
208 member3Actor.clear();
209 member3Actor.expectMessageClass(RequestVoteReply.class, 1);
210 member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
212 member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
214 member1Actor.waitForExpectedMessages(RequestVote.class);
215 member2Actor.waitForExpectedMessages(RequestVote.class);
216 member3Actor.waitForExpectedMessages(RequestVoteReply.class);
218 RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
219 assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
220 assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
222 // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
223 // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
224 // AppendEntries message to member 2 is dropped.
226 member1Actor.waitForExpectedMessages(AppendEntries.class);
227 member2Actor.waitForExpectedMessages(AppendEntries.class);
228 member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
230 verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
231 verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
232 verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
234 assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
235 assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
236 assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
238 testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
241 private void sendInitialElectionTimeoutToFollowerMember2() {
242 testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
244 // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
245 // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
246 // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
247 // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
248 // simulate loss of network connectivity between members 2 and 3.
250 member1Actor.expectMessageClass(RequestVote.class, 1);
252 member2Actor.expectBehaviorStateChange();
254 member3Actor.dropMessagesToBehavior(RequestVote.class);
256 member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
258 member1Actor.waitForExpectedMessages(RequestVote.class);
259 member3Actor.waitForExpectedMessages(RequestVote.class);
261 // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
262 // term. It won't send back a RequestVoteReply in this case.
264 verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
266 // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
268 member2Actor.waitForBehaviorStateChange();
269 verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
271 assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
272 assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
273 assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
275 testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
278 private void setupInitialMemberBehaviors() throws Exception {
279 testLog.info("setupInitialMemberBehaviors starting");
281 // Create member 2's behavior initially as Follower
283 member2Context = newRaftActorContext("member2", member2ActorRef,
284 ImmutableMap.<String,String>builder()
285 .put("member1", member1ActorRef.path().toString())
286 .put("member3", member3ActorRef.path().toString()).build());
288 DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
289 member2Context.setConfigParams(member2ConfigParams);
291 member2Actor.behavior = new Follower(member2Context);
292 member2Context.setCurrentBehavior(member2Actor.behavior);
294 // Create member 3's behavior initially as Follower
296 member3Context = newRaftActorContext("member3", member3ActorRef,
297 ImmutableMap.<String,String>builder()
298 .put("member1", member1ActorRef.path().toString())
299 .put("member2", member2ActorRef.path().toString()).build());
301 DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
302 member3Context.setConfigParams(member3ConfigParams);
304 member3Actor.behavior = new Follower(member3Context);
305 member3Context.setCurrentBehavior(member3Actor.behavior);
307 // Create member 1's behavior initially as Leader
309 member1Context = newRaftActorContext("member1", member1ActorRef,
310 ImmutableMap.<String,String>builder()
311 .put("member2", member2ActorRef.path().toString())
312 .put("member3", member3ActorRef.path().toString()).build());
314 DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
315 member1Context.setConfigParams(member1ConfigParams);
317 initializeLeaderBehavior(member1Actor, member1Context, 2);
319 member2Actor.clear();
320 member3Actor.clear();
322 testLog.info("setupInitialMemberBehaviors ending");