opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11
  12 import akka.actor.ActorRef;
  13 import com.google.common.collect.ImmutableMap;
  14 import org.junit.Test;
  15 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  16 import org.opendaylight.controller.cluster.raft.RaftState;
  17 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  18 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
  19 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  20 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  21 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  22 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  23
  24 /**
  25  * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
  26  *
  27  * @author Thomas Pantelis
  28  */
  29 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  30
  31     /**
  32      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
  33      * sends a heartbeat first when connectivity is re-established.
  34      */
  35     @Test
  36     public void runTest1() throws Exception {
  37         testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
  38
  39         setupInitialMemberBehaviors();
  40
  41         sendInitialElectionTimeoutToFollowerMember2();
  42
  43         sendInitialElectionTimeoutToFollowerMember3();
  44
  45         sendElectionTimeoutToNowCandidateMember2();
  46
  47         resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
  48
  49         testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
  50     }
  51
  52     /**
  53      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
  54      * sends a heartbeat first when connectivity is re-established.
  55      */
  56     @Test
  57     public void runTest2() throws Exception {
  58         testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
  59
  60         setupInitialMemberBehaviors();
  61
  62         sendInitialElectionTimeoutToFollowerMember2();
  63
  64         sendInitialElectionTimeoutToFollowerMember3();
  65
  66         sendElectionTimeoutToNowCandidateMember2();
  67
  68         resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
  69
  70         testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
  71     }
  72
  73     private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
  74         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
  75
  76         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
  77         // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
  78         // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
  79         // Follower as its term is less than member 2's.
  80
  81         member1Actor.clear();
  82         member1Actor.expectMessageClass(AppendEntries.class, 1);
  83
  84         member2Actor.clear();
  85         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
  86
  87         member3Actor.clear();
  88         member3Actor.expectMessageClass(AppendEntries.class, 1);
  89
  90         sendHeartbeat(member2ActorRef);
  91
  92         member1Actor.waitForExpectedMessages(AppendEntries.class);
  93         member3Actor.waitForExpectedMessages(AppendEntries.class);
  94
  95         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
  96
  97         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  98         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
  99         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 100
 101         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 102         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 103         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 104
 105         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
 106     }
 107
 108     private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
 109         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
 110
 111         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
 112         // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
 113         // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
 114         // This should cause member 3 to switch to Follower.
 115
 116         member1Actor.clear();
 117         member1Actor.expectMessageClass(AppendEntries.class, 1);
 118
 119         member2Actor.clear();
 120         member2Actor.expectMessageClass(AppendEntries.class, 1);
 121
 122         member3Actor.clear();
 123         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 124
 125         sendHeartbeat(member3ActorRef);
 126
 127         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 128
 129         AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
 130         assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
 131         assertEquals("getTerm", 3, appendEntriesReply.getTerm());
 132
 133         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 134         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 135         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 136
 137         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 138         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 139         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 140
 141         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
 142     }
 143
 144     private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
 145         testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
 146
 147         // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
 148         // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
 149         // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
 150         // are dropped to simulate loss of network connectivity. Note member 2 will increment its
 151         // election term to 3.
 152
 153         member1Actor.clear();
 154         member1Actor.expectMessageClass(AppendEntries.class, 1);
 155
 156         member2Actor.clear();
 157         member2Actor.expectMessageClass(RequestVoteReply.class, 1);
 158         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
 159
 160         member3Actor.clear();
 161         member3Actor.dropMessagesToBehavior(AppendEntries.class);
 162         member3Actor.dropMessagesToBehavior(RequestVote.class);
 163
 164         member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
 165
 166         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
 167
 168         RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
 169         assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 170         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 171
 172         member3Actor.waitForExpectedMessages(RequestVote.class);
 173
 174         member1Actor.waitForExpectedMessages(AppendEntries.class);
 175         member3Actor.waitForExpectedMessages(AppendEntries.class);
 176         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
 177
 178         // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
 179         // is 3 and member 3's term is 2.
 180
 181         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 182         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 183         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 184
 185         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 186         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 187         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 188
 189         testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
 190     }
 191
 192     private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
 193         testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
 194
 195         // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
 196         // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
 197         // should reply and grant the vote but member 2 will drop the message to simulate loss of network
 198         // connectivity between members 2 and 3. member 3 should switch to leader.
 199
 200         member1Actor.clear();
 201         member1Actor.expectMessageClass(RequestVote.class, 1);
 202         member1Actor.expectMessageClass(AppendEntries.class, 1);
 203
 204         member2Actor.clear();
 205         member2Actor.dropMessagesToBehavior(RequestVote.class);
 206         member2Actor.dropMessagesToBehavior(AppendEntries.class);
 207
 208         member3Actor.clear();
 209         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
 210         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 211
 212         member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 213
 214         member1Actor.waitForExpectedMessages(RequestVote.class);
 215         member2Actor.waitForExpectedMessages(RequestVote.class);
 216         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
 217
 218         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
 219         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 220         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 221
 222         // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
 223         // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
 224         // AppendEntries message to member 2 is dropped.
 225
 226         member1Actor.waitForExpectedMessages(AppendEntries.class);
 227         member2Actor.waitForExpectedMessages(AppendEntries.class);
 228         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 229
 230         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 231         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 232         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 233
 234         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 235         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 236         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 237
 238         testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
 239     }
 240
 241     private void sendInitialElectionTimeoutToFollowerMember2() {
 242         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 243
 244         // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
 245         // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
 246         // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
 247         // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
 248         // simulate loss of network connectivity between members 2 and 3.
 249
 250         member1Actor.expectMessageClass(RequestVote.class, 1);
 251
 252         member2Actor.expectBehaviorStateChange();
 253
 254         member3Actor.dropMessagesToBehavior(RequestVote.class);
 255
 256         member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 257
 258         member1Actor.waitForExpectedMessages(RequestVote.class);
 259         member3Actor.waitForExpectedMessages(RequestVote.class);
 260
 261         // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
 262         // term. It won't send back a RequestVoteReply in this case.
 263
 264         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 265
 266         // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
 267
 268         member2Actor.waitForBehaviorStateChange();
 269         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 270
 271         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 272         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 273         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 274
 275         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 276     }
 277
 278     private void setupInitialMemberBehaviors() throws Exception {
 279         testLog.info("setupInitialMemberBehaviors starting");
 280
 281         // Create member 2's behavior initially as Follower
 282
 283         member2Context = newRaftActorContext("member2", member2ActorRef,
 284                 ImmutableMap.<String,String>builder()
 285                     .put("member1", member1ActorRef.path().toString())
 286                     .put("member3", member3ActorRef.path().toString()).build());
 287
 288         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 289         member2Context.setConfigParams(member2ConfigParams);
 290
 291         member2Actor.behavior = new Follower(member2Context);
 292         member2Context.setCurrentBehavior(member2Actor.behavior);
 293
 294         // Create member 3's behavior initially as Follower
 295
 296         member3Context = newRaftActorContext("member3", member3ActorRef,
 297                 ImmutableMap.<String,String>builder()
 298                     .put("member1", member1ActorRef.path().toString())
 299                     .put("member2", member2ActorRef.path().toString()).build());
 300
 301         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 302         member3Context.setConfigParams(member3ConfigParams);
 303
 304         member3Actor.behavior = new Follower(member3Context);
 305         member3Context.setCurrentBehavior(member3Actor.behavior);
 306
 307         // Create member 1's behavior initially as Leader
 308
 309         member1Context = newRaftActorContext("member1", member1ActorRef,
 310                 ImmutableMap.<String,String>builder()
 311                     .put("member2", member2ActorRef.path().toString())
 312                     .put("member3", member3ActorRef.path().toString()).build());
 313
 314         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 315         member1Context.setConfigParams(member1ConfigParams);
 316
 317         initializeLeaderBehavior(member1Actor, member1Context, 2);
 318
 319         member2Actor.clear();
 320         member3Actor.clear();
 321
 322         testLog.info("setupInitialMemberBehaviors ending");
 323     }
 324 }