Fix resource leaks in test cases
[controller.git] / opendaylight / md-sal / sal-akka-raft / src / test / java / org / opendaylight / controller / cluster / raft / behaviors / PartitionedLeadersElectionScenarioTest.java
1 /*
2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.raft.behaviors;
9
10 import static org.junit.Assert.assertEquals;
11 import akka.actor.ActorRef;
12 import com.google.common.collect.ImmutableMap;
13 import org.junit.Test;
14 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
15 import org.opendaylight.controller.cluster.raft.RaftState;
16 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
17 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
18 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
19 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
20 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
21
22 /**
23  * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
24  *
25  * @author Thomas Pantelis
26  */
27 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
28
29     /**
30      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
31      * sends a heartbeat first when connectivity is re-established.
32      */
33     @Test
34     public void runTest1() throws Exception {
35         testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
36
37         setupInitialMemberBehaviors();
38
39         sendInitialElectionTimeoutToFollowerMember2();
40
41         sendInitialElectionTimeoutToFollowerMember3();
42
43         sendElectionTimeoutToNowCandidateMember2();
44
45         resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
46
47         testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
48     }
49
50     /**
51      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
52      * sends a heartbeat first when connectivity is re-established.
53      */
54     @Test
55     public void runTest2() throws Exception {
56         testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
57
58         setupInitialMemberBehaviors();
59
60         sendInitialElectionTimeoutToFollowerMember2();
61
62         sendInitialElectionTimeoutToFollowerMember3();
63
64         sendElectionTimeoutToNowCandidateMember2();
65
66         resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
67
68         testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
69     }
70
71     private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
72         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
73
74         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
75         // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
76         // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
77         // Follower as its term is less than member 2's.
78
79         member1Actor.clear();
80         member1Actor.expectMessageClass(AppendEntries.class, 1);
81
82         member2Actor.clear();
83         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
84
85         member3Actor.clear();
86         member3Actor.expectMessageClass(AppendEntries.class, 1);
87
88         sendHeartbeat(member2ActorRef);
89
90         member1Actor.waitForExpectedMessages(AppendEntries.class);
91         member3Actor.waitForExpectedMessages(AppendEntries.class);
92
93         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
94
95         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
96         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
97         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
98
99         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
100         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
101         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
102
103         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
104     }
105
106     private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
107         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
108
109         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
110         // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
111         // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
112         // This should cause member 3 to switch to Follower.
113
114         member1Actor.clear();
115         member1Actor.expectMessageClass(AppendEntries.class, 1);
116
117         member2Actor.clear();
118         member2Actor.expectMessageClass(AppendEntries.class, 1);
119
120         member3Actor.clear();
121         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
122
123         sendHeartbeat(member3ActorRef);
124
125         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
126
127         AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
128         assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
129         assertEquals("getTerm", 3, appendEntriesReply.getTerm());
130
131         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
132         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
133         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
134
135         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
136         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
137         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
138
139         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
140     }
141
142     private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
143         testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
144
145         // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
146         // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
147         // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
148         // are dropped to simulate loss of network connectivity. Note member 2 will increment its
149         // election term to 3.
150
151         member1Actor.clear();
152         member1Actor.expectMessageClass(AppendEntries.class, 1);
153
154         member2Actor.clear();
155         member2Actor.expectMessageClass(RequestVoteReply.class, 1);
156         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
157
158         member3Actor.clear();
159         member3Actor.dropMessagesToBehavior(AppendEntries.class);
160         member3Actor.dropMessagesToBehavior(RequestVote.class);
161
162         member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
163
164         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
165
166         RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
167         assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
168         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
169
170         member3Actor.waitForExpectedMessages(RequestVote.class);
171
172         member1Actor.waitForExpectedMessages(AppendEntries.class);
173         member3Actor.waitForExpectedMessages(AppendEntries.class);
174         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
175
176         // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
177         // is 3 and member 3's term is 2.
178
179         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
180         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
181         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
182
183         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
184         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
185         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
186
187         testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
188     }
189
190     private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
191         testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
192
193         // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
194         // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
195         // should reply and grant the vote but member 2 will drop the message to simulate loss of network
196         // connectivity between members 2 and 3. member 3 should switch to leader.
197
198         member1Actor.clear();
199         member1Actor.expectMessageClass(RequestVote.class, 1);
200         member1Actor.expectMessageClass(AppendEntries.class, 1);
201
202         member2Actor.clear();
203         member2Actor.dropMessagesToBehavior(RequestVote.class);
204         member2Actor.dropMessagesToBehavior(AppendEntries.class);
205
206         member3Actor.clear();
207         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
208         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
209
210         member3ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
211
212         member1Actor.waitForExpectedMessages(RequestVote.class);
213         member2Actor.waitForExpectedMessages(RequestVote.class);
214         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
215
216         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
217         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
218         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
219
220         // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
221         // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
222         // AppendEntries message to member 2 is dropped.
223
224         member1Actor.waitForExpectedMessages(AppendEntries.class);
225         member2Actor.waitForExpectedMessages(AppendEntries.class);
226         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
227
228         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
229         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
230         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
231
232         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
233         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
234         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
235
236         testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
237     }
238
239     private void sendInitialElectionTimeoutToFollowerMember2() {
240         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
241
242         // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
243         // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
244         // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
245         // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
246         // simulate loss of network connectivity between members 2 and 3.
247
248         member1Actor.expectMessageClass(RequestVote.class, 1);
249
250         member2Actor.expectBehaviorStateChange();
251
252         member3Actor.dropMessagesToBehavior(RequestVote.class);
253
254         member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
255
256         member1Actor.waitForExpectedMessages(RequestVote.class);
257         member3Actor.waitForExpectedMessages(RequestVote.class);
258
259         // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
260         // term. It won't send back a RequestVoteReply in this case.
261
262         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
263
264         // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
265
266         member2Actor.waitForBehaviorStateChange();
267         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
268
269         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
270         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
271         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
272
273         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
274     }
275
276     private void setupInitialMemberBehaviors() throws Exception {
277         testLog.info("setupInitialMemberBehaviors starting");
278
279         // Create member 2's behavior initially as Follower
280
281         member2Context = newRaftActorContext("member2", member2ActorRef,
282                 ImmutableMap.<String,String>builder().
283                     put("member1", member1ActorRef.path().toString()).
284                     put("member3", member3ActorRef.path().toString()).build());
285
286         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
287         member2Context.setConfigParams(member2ConfigParams);
288
289         member2Actor.behavior = new Follower(member2Context);
290
291         // Create member 3's behavior initially as Follower
292
293         member3Context = newRaftActorContext("member3", member3ActorRef,
294                 ImmutableMap.<String,String>builder().
295                     put("member1", member1ActorRef.path().toString()).
296                     put("member2", member2ActorRef.path().toString()).build());
297
298         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
299         member3Context.setConfigParams(member3ConfigParams);
300
301         member3Actor.behavior = new Follower(member3Context);
302
303         // Create member 1's behavior initially as Leader
304
305         member1Context = newRaftActorContext("member1", member1ActorRef,
306                 ImmutableMap.<String,String>builder().
307                     put("member2", member2ActorRef.path().toString()).
308                     put("member3", member3ActorRef.path().toString()).build());
309
310         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
311         member1Context.setConfigParams(member1ConfigParams);
312
313         initializeLeaderBehavior(member1Actor, member1Context, 2);
314
315         member2Actor.clear();
316         member3Actor.clear();
317
318         testLog.info("setupInitialMemberBehaviors ending");
319     }
320 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.