opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import akka.actor.ActorRef;
  12 import com.google.common.collect.ImmutableMap;
  13 import org.junit.Test;
  14 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  15 import org.opendaylight.controller.cluster.raft.RaftState;
  16 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  17 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  18 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  19 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  20 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  21
  22 /**
  23  * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
  24  *
  25  * @author Thomas Pantelis
  26  */
  27 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  28
  29     /**
  30      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
  31      * sends a heartbeat first when connectivity is re-established.
  32      */
  33     @Test
  34     public void runTest1() throws Exception {
  35         testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
  36
  37         setupInitialMemberBehaviors();
  38
  39         sendInitialElectionTimeoutToFollowerMember2();
  40
  41         sendInitialElectionTimeoutToFollowerMember3();
  42
  43         sendElectionTimeoutToNowCandidateMember2();
  44
  45         resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
  46
  47         testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
  48     }
  49
  50     /**
  51      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
  52      * sends a heartbeat first when connectivity is re-established.
  53      */
  54     @Test
  55     public void runTest2() throws Exception {
  56         testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
  57
  58         setupInitialMemberBehaviors();
  59
  60         sendInitialElectionTimeoutToFollowerMember2();
  61
  62         sendInitialElectionTimeoutToFollowerMember3();
  63
  64         sendElectionTimeoutToNowCandidateMember2();
  65
  66         resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
  67
  68         testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
  69     }
  70
  71     private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
  72         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
  73
  74         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
  75         // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
  76         // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
  77         // Follower as its term is less than member 2's.
  78
  79         member1Actor.clear();
  80         member1Actor.expectMessageClass(AppendEntries.class, 1);
  81
  82         member2Actor.clear();
  83         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
  84
  85         member3Actor.clear();
  86         member3Actor.expectMessageClass(AppendEntries.class, 1);
  87
  88         sendHeartbeat(member2ActorRef);
  89
  90         member1Actor.waitForExpectedMessages(AppendEntries.class);
  91         member3Actor.waitForExpectedMessages(AppendEntries.class);
  92
  93         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
  94
  95         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  96         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
  97         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
  98
  99         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 100         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 101         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 102
 103         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
 104     }
 105
 106     private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
 107         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
 108
 109         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
 110         // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
 111         // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
 112         // This should cause member 3 to switch to Follower.
 113
 114         member1Actor.clear();
 115         member1Actor.expectMessageClass(AppendEntries.class, 1);
 116
 117         member2Actor.clear();
 118         member2Actor.expectMessageClass(AppendEntries.class, 1);
 119
 120         member3Actor.clear();
 121         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 122
 123         sendHeartbeat(member3ActorRef);
 124
 125         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 126
 127         AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
 128         assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
 129         assertEquals("getTerm", 3, appendEntriesReply.getTerm());
 130
 131         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 132         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 133         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 134
 135         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 136         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 137         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 138
 139         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
 140     }
 141
 142     private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
 143         testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
 144
 145         // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
 146         // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
 147         // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
 148         // are dropped to simulate loss of network connectivity. Note member 2 will increment its
 149         // election term to 3.
 150
 151         member1Actor.clear();
 152         member1Actor.expectMessageClass(AppendEntries.class, 1);
 153
 154         member2Actor.clear();
 155         member2Actor.expectMessageClass(RequestVoteReply.class, 1);
 156         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
 157
 158         member3Actor.clear();
 159         member3Actor.dropMessagesToBehavior(AppendEntries.class);
 160         member3Actor.dropMessagesToBehavior(RequestVote.class);
 161
 162         member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
 163
 164         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
 165
 166         RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
 167         assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 168         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 169
 170         member3Actor.waitForExpectedMessages(RequestVote.class);
 171
 172         member1Actor.waitForExpectedMessages(AppendEntries.class);
 173         member3Actor.waitForExpectedMessages(AppendEntries.class);
 174         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
 175
 176         // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
 177         // is 3 and member 3's term is 2.
 178
 179         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 180         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 181         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 182
 183         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 184         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 185         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 186
 187         testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
 188     }
 189
 190     private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
 191         testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
 192
 193         // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
 194         // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
 195         // should reply and grant the vote but member 2 will drop the message to simulate loss of network
 196         // connectivity between members 2 and 3. member 3 should switch to leader.
 197
 198         member1Actor.clear();
 199         member1Actor.expectMessageClass(RequestVote.class, 1);
 200         member1Actor.expectMessageClass(AppendEntries.class, 1);
 201
 202         member2Actor.clear();
 203         member2Actor.dropMessagesToBehavior(RequestVote.class);
 204         member2Actor.dropMessagesToBehavior(AppendEntries.class);
 205
 206         member3Actor.clear();
 207         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
 208         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 209
 210         member3ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
 211
 212         member1Actor.waitForExpectedMessages(RequestVote.class);
 213         member2Actor.waitForExpectedMessages(RequestVote.class);
 214         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
 215
 216         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
 217         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 218         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 219
 220         // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
 221         // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
 222         // AppendEntries message to member 2 is dropped.
 223
 224         member1Actor.waitForExpectedMessages(AppendEntries.class);
 225         member2Actor.waitForExpectedMessages(AppendEntries.class);
 226         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 227
 228         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 229         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 230         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 231
 232         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 233         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 234         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 235
 236         testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
 237     }
 238
 239     private void sendInitialElectionTimeoutToFollowerMember2() {
 240         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 241
 242         // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
 243         // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
 244         // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
 245         // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
 246         // simulate loss of network connectivity between members 2 and 3.
 247
 248         member1Actor.expectMessageClass(RequestVote.class, 1);
 249
 250         member2Actor.expectBehaviorStateChange();
 251
 252         member3Actor.dropMessagesToBehavior(RequestVote.class);
 253
 254         member2ActorRef.tell(new ElectionTimeout(), ActorRef.noSender());
 255
 256         member1Actor.waitForExpectedMessages(RequestVote.class);
 257         member3Actor.waitForExpectedMessages(RequestVote.class);
 258
 259         // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
 260         // term. It won't send back a RequestVoteReply in this case.
 261
 262         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 263
 264         // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
 265
 266         member2Actor.waitForBehaviorStateChange();
 267         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 268
 269         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 270         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 271         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 272
 273         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 274     }
 275
 276     private void setupInitialMemberBehaviors() throws Exception {
 277         testLog.info("setupInitialMemberBehaviors starting");
 278
 279         // Create member 2's behavior initially as Follower
 280
 281         member2Context = newRaftActorContext("member2", member2ActorRef,
 282                 ImmutableMap.<String,String>builder().
 283                     put("member1", member1ActorRef.path().toString()).
 284                     put("member3", member3ActorRef.path().toString()).build());
 285
 286         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 287         member2Context.setConfigParams(member2ConfigParams);
 288
 289         Follower member2Behavior = new Follower(member2Context);
 290         member2Actor.behavior = member2Behavior;
 291
 292         // Create member 3's behavior initially as Follower
 293
 294         member3Context = newRaftActorContext("member3", member3ActorRef,
 295                 ImmutableMap.<String,String>builder().
 296                     put("member1", member1ActorRef.path().toString()).
 297                     put("member2", member2ActorRef.path().toString()).build());
 298
 299         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 300         member3Context.setConfigParams(member3ConfigParams);
 301
 302         Follower member3Behavior = new Follower(member3Context);
 303         member3Actor.behavior = member3Behavior;
 304
 305         // Create member 1's behavior initially as Leader
 306
 307         member1Context = newRaftActorContext("member1", member1ActorRef,
 308                 ImmutableMap.<String,String>builder().
 309                     put("member2", member2ActorRef.path().toString()).
 310                     put("member3", member3ActorRef.path().toString()).build());
 311
 312         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 313         member1Context.setConfigParams(member1ConfigParams);
 314
 315         initializeLeaderBehavior(member1Actor, member1Context, 2);
 316
 317         member2Actor.clear();
 318         member3Actor.clear();
 319
 320         testLog.info("setupInitialMemberBehaviors ending");
 321     }
 322 }