cf02a1f23e8a9343295186b08b6fae134ae242ea
[controller.git] / opendaylight / md-sal / sal-akka-raft / src / test / java / org / opendaylight / controller / cluster / raft / behaviors / PartitionedLeadersElectionScenarioTest.java
1 /*
2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.raft.behaviors;
9
10 import static org.junit.Assert.assertEquals;
11
12 import akka.actor.ActorRef;
13 import com.google.common.collect.ImmutableMap;
14 import org.junit.Test;
15 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
16 import org.opendaylight.controller.cluster.raft.RaftState;
17 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
18 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
19 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
20 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
21 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
22 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
23
24 /**
25  * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
26  *
27  * @author Thomas Pantelis
28  */
29 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
30
31     /**
32      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
33      * sends a heartbeat first when connectivity is re-established.
34      */
35     @Test
36     public void runTest1() throws Exception {
37         testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
38
39         setupInitialMemberBehaviors();
40
41         sendInitialElectionTimeoutToFollowerMember2();
42
43         sendInitialElectionTimeoutToFollowerMember3();
44
45         sendElectionTimeoutToNowCandidateMember2();
46
47         resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
48
49         testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
50     }
51
52     /**
53      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
54      * sends a heartbeat first when connectivity is re-established.
55      */
56     @Test
57     public void runTest2() throws Exception {
58         testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
59
60         setupInitialMemberBehaviors();
61
62         sendInitialElectionTimeoutToFollowerMember2();
63
64         sendInitialElectionTimeoutToFollowerMember3();
65
66         sendElectionTimeoutToNowCandidateMember2();
67
68         resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
69
70         testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
71     }
72
73     private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
74         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
75
76         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
77         // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
78         // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
79         // Follower as its term is less than member 2's.
80
81         member1Actor.clear();
82         member1Actor.expectMessageClass(AppendEntries.class, 1);
83
84         member2Actor.clear();
85         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
86
87         member3Actor.clear();
88         member3Actor.expectMessageClass(AppendEntries.class, 1);
89
90         sendHeartbeat(member2ActorRef);
91
92         member1Actor.waitForExpectedMessages(AppendEntries.class);
93         member3Actor.waitForExpectedMessages(AppendEntries.class);
94
95         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
96
97         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
98         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
99         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
100
101         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
102         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
103         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
104
105         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
106     }
107
108     private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
109         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
110
111         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
112         // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
113         // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
114         // This should cause member 3 to switch to Follower.
115
116         member1Actor.clear();
117         member1Actor.expectMessageClass(AppendEntries.class, 1);
118
119         member2Actor.clear();
120         member2Actor.expectMessageClass(AppendEntries.class, 1);
121
122         member3Actor.clear();
123         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
124
125         sendHeartbeat(member3ActorRef);
126
127         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
128
129         AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
130         assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
131         assertEquals("getTerm", 3, appendEntriesReply.getTerm());
132
133         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
134         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
135         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
136
137         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
138         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
139         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
140
141         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
142     }
143
144     private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
145         testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
146
147         // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
148         // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
149         // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
150         // are dropped to simulate loss of network connectivity. Note member 2 will increment its
151         // election term to 3.
152
153         member1Actor.clear();
154         member1Actor.expectMessageClass(AppendEntries.class, 1);
155
156         member2Actor.clear();
157         member2Actor.expectMessageClass(RequestVoteReply.class, 1);
158         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
159
160         member3Actor.clear();
161         member3Actor.dropMessagesToBehavior(AppendEntries.class);
162         member3Actor.dropMessagesToBehavior(RequestVote.class);
163
164         member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
165
166         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
167
168         RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
169         assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
170         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
171
172         member3Actor.waitForExpectedMessages(RequestVote.class);
173
174         member1Actor.waitForExpectedMessages(AppendEntries.class);
175         member3Actor.waitForExpectedMessages(AppendEntries.class);
176         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
177
178         // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
179         // is 3 and member 3's term is 2.
180
181         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
182         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
183         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
184
185         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
186         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
187         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
188
189         testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
190     }
191
192     private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
193         testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
194
195         // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
196         // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
197         // should reply and grant the vote but member 2 will drop the message to simulate loss of network
198         // connectivity between members 2 and 3. member 3 should switch to leader.
199
200         member1Actor.clear();
201         member1Actor.expectMessageClass(RequestVote.class, 1);
202         member1Actor.expectMessageClass(AppendEntries.class, 1);
203
204         member2Actor.clear();
205         member2Actor.dropMessagesToBehavior(RequestVote.class);
206         member2Actor.dropMessagesToBehavior(AppendEntries.class);
207
208         member3Actor.clear();
209         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
210         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
211
212         member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
213
214         member1Actor.waitForExpectedMessages(RequestVote.class);
215         member2Actor.waitForExpectedMessages(RequestVote.class);
216         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
217
218         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
219         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
220         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
221
222         // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
223         // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
224         // AppendEntries message to member 2 is dropped.
225
226         member1Actor.waitForExpectedMessages(AppendEntries.class);
227         member2Actor.waitForExpectedMessages(AppendEntries.class);
228         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
229
230         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
231         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
232         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
233
234         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
235         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
236         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
237
238         testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
239     }
240
241     private void sendInitialElectionTimeoutToFollowerMember2() {
242         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
243
244         // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
245         // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
246         // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
247         // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
248         // simulate loss of network connectivity between members 2 and 3.
249
250         member1Actor.expectMessageClass(RequestVote.class, 1);
251
252         member2Actor.expectBehaviorStateChange();
253
254         member3Actor.dropMessagesToBehavior(RequestVote.class);
255
256         member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
257
258         member1Actor.waitForExpectedMessages(RequestVote.class);
259         member3Actor.waitForExpectedMessages(RequestVote.class);
260
261         // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
262         // term. It won't send back a RequestVoteReply in this case.
263
264         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
265
266         // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
267
268         member2Actor.waitForBehaviorStateChange();
269         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
270
271         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
272         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
273         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
274
275         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
276     }
277
278     private void setupInitialMemberBehaviors() throws Exception {
279         testLog.info("setupInitialMemberBehaviors starting");
280
281         // Create member 2's behavior initially as Follower
282
283         member2Context = newRaftActorContext("member2", member2ActorRef,
284                 ImmutableMap.<String,String>builder()
285                     .put("member1", member1ActorRef.path().toString())
286                     .put("member3", member3ActorRef.path().toString()).build());
287
288         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
289         member2Context.setConfigParams(member2ConfigParams);
290
291         member2Actor.self().tell(new SetBehavior(new Follower(member2Context), member2Context),
292                 ActorRef.noSender());
293
294         // Create member 3's behavior initially as Follower
295
296         member3Context = newRaftActorContext("member3", member3ActorRef,
297                 ImmutableMap.<String,String>builder()
298                     .put("member1", member1ActorRef.path().toString())
299                     .put("member2", member2ActorRef.path().toString()).build());
300
301         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
302         member3Context.setConfigParams(member3ConfigParams);
303
304         member3Actor.self().tell(new SetBehavior(new Follower(member3Context), member3Context),
305                 ActorRef.noSender());
306
307         // Create member 1's behavior initially as Leader
308
309         member1Context = newRaftActorContext("member1", member1ActorRef,
310                 ImmutableMap.<String,String>builder()
311                     .put("member2", member2ActorRef.path().toString())
312                     .put("member3", member3ActorRef.path().toString()).build());
313
314         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
315         member1Context.setConfigParams(member1ConfigParams);
316
317         initializeLeaderBehavior(member1Actor, member1Context, 2);
318
319         member2Actor.clear();
320         member3Actor.clear();
321
322         testLog.info("setupInitialMemberBehaviors ending");
323     }
324 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.