opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/PartitionedLeadersElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import akka.actor.ActorRef;
  12 import com.google.common.collect.ImmutableMap;
  13 import org.junit.Test;
  14 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  15 import org.opendaylight.controller.cluster.raft.RaftState;
  16 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  17 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
  18 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  19 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  20 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  21 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  22
  23 /**
  24  * A leader election scenario test that causes partitioned leaders by dropping messages between 2 members.
  25  *
  26  * @author Thomas Pantelis
  27  */
  28 public class PartitionedLeadersElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  29
  30     /**
  31      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 3
  32      * sends a heartbeat first when connectivity is re-established.
  33      */
  34     @Test
  35     public void runTest1() throws Exception {
  36         testLog.info("PartitionedLeadersElectionScenarioTest 1 starting");
  37
  38         setupInitialMemberBehaviors();
  39
  40         sendInitialElectionTimeoutToFollowerMember2();
  41
  42         sendInitialElectionTimeoutToFollowerMember3();
  43
  44         sendElectionTimeoutToNowCandidateMember2();
  45
  46         resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst();
  47
  48         testLog.info("PartitionedLeadersElectionScenarioTest 1 ending");
  49     }
  50
  51     /**
  52      * This test sets up a scenario with partitioned leaders member 2 and 3 where partitioned leader 2
  53      * sends a heartbeat first when connectivity is re-established.
  54      */
  55     @Test
  56     public void runTest2() throws Exception {
  57         testLog.info("PartitionedLeadersElectionScenarioTest 2 starting");
  58
  59         setupInitialMemberBehaviors();
  60
  61         sendInitialElectionTimeoutToFollowerMember2();
  62
  63         sendInitialElectionTimeoutToFollowerMember3();
  64
  65         sendElectionTimeoutToNowCandidateMember2();
  66
  67         resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst();
  68
  69         testLog.info("PartitionedLeadersElectionScenarioTest 2 ending");
  70     }
  71
  72     private void resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst() {
  73         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst starting");
  74
  75         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
  76         // the 2. Send heartbeats (AppendEntries) from partitioned leader member 2. Follower member 1 should
  77         // return a successful AppendEntriesReply b/c its term matches member 2's. member 3 should switch to
  78         // Follower as its term is less than member 2's.
  79
  80         member1Actor.clear();
  81         member1Actor.expectMessageClass(AppendEntries.class, 1);
  82
  83         member2Actor.clear();
  84         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
  85
  86         member3Actor.clear();
  87         member3Actor.expectMessageClass(AppendEntries.class, 1);
  88
  89         sendHeartbeat(member2ActorRef);
  90
  91         member1Actor.waitForExpectedMessages(AppendEntries.class);
  92         member3Actor.waitForExpectedMessages(AppendEntries.class);
  93
  94         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
  95
  96         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  97         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
  98         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
  99
 100         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 101         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 102         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 103
 104         testLog.info("resolvePartitionedLeadersWithLeaderMember2SendingHeartbeatFirst ending");
 105     }
 106
 107     private void resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst() throws Exception {
 108         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst starting");
 109
 110         // Re-establish connectivity between member 2 and 3, ie stop dropping messages between
 111         // the 2. Send heartbeats (AppendEntries) from now leader member 3. Both member 1 and 2 should send
 112         // back an unsuccessful AppendEntriesReply b/c their term (3) is greater than member 3's term (2).
 113         // This should cause member 3 to switch to Follower.
 114
 115         member1Actor.clear();
 116         member1Actor.expectMessageClass(AppendEntries.class, 1);
 117
 118         member2Actor.clear();
 119         member2Actor.expectMessageClass(AppendEntries.class, 1);
 120
 121         member3Actor.clear();
 122         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 123
 124         sendHeartbeat(member3ActorRef);
 125
 126         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 127
 128         AppendEntriesReply appendEntriesReply = member3Actor.getCapturedMessage(AppendEntriesReply.class);
 129         assertEquals("isSuccess", false, appendEntriesReply.isSuccess());
 130         assertEquals("getTerm", 3, appendEntriesReply.getTerm());
 131
 132         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 133         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 134         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 135
 136         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 137         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 138         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 139
 140         testLog.info("resolvePartitionedLeadersWithLeaderMember3SendingHeartbeatFirst ending");
 141     }
 142
 143     private void sendElectionTimeoutToNowCandidateMember2() throws Exception {
 144         testLog.info("sendElectionTimeoutToNowCandidateMember2 starting");
 145
 146         // member 2, now a candidate, is partitioned from the Leader (now member 3) and hasn't received any
 147         // messages. It would get another ElectionTimeout so simulate that. member 1 should send back a reply
 148         // granting the vote. Messages (RequestVote and AppendEntries) from member 2 to member 3
 149         // are dropped to simulate loss of network connectivity. Note member 2 will increment its
 150         // election term to 3.
 151
 152         member1Actor.clear();
 153         member1Actor.expectMessageClass(AppendEntries.class, 1);
 154
 155         member2Actor.clear();
 156         member2Actor.expectMessageClass(RequestVoteReply.class, 1);
 157         member2Actor.expectMessageClass(AppendEntriesReply.class, 1);
 158
 159         member3Actor.clear();
 160         member3Actor.dropMessagesToBehavior(AppendEntries.class);
 161         member3Actor.dropMessagesToBehavior(RequestVote.class);
 162
 163         member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
 164
 165         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
 166
 167         RequestVoteReply requestVoteReply = member2Actor.getCapturedMessage(RequestVoteReply.class);
 168         assertEquals("getTerm", member2Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 169         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 170
 171         member3Actor.waitForExpectedMessages(RequestVote.class);
 172
 173         member1Actor.waitForExpectedMessages(AppendEntries.class);
 174         member3Actor.waitForExpectedMessages(AppendEntries.class);
 175         member2Actor.waitForExpectedMessages(AppendEntriesReply.class);
 176
 177         // We end up with 2 partitioned leaders both leading member 1. The term for member 1 and 3
 178         // is 3 and member 3's term is 2.
 179
 180         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 181         verifyBehaviorState("member 2", member2Actor, RaftState.Leader);
 182         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 183
 184         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 185         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 186         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 187
 188         testLog.info("sendElectionTimeoutToNowCandidateMember2 ending");
 189     }
 190
 191     private void sendInitialElectionTimeoutToFollowerMember3() throws Exception {
 192         testLog.info("sendInitialElectionTimeoutToFollowerMember3 starting");
 193
 194         // Send ElectionTimeout to member 3 to simulate no heartbeat from a Leader (originally member 1).
 195         // member 3 should switch to Candidate and send out RequestVote messages. member 1, now a follower,
 196         // should reply and grant the vote but member 2 will drop the message to simulate loss of network
 197         // connectivity between members 2 and 3. member 3 should switch to leader.
 198
 199         member1Actor.clear();
 200         member1Actor.expectMessageClass(RequestVote.class, 1);
 201         member1Actor.expectMessageClass(AppendEntries.class, 1);
 202
 203         member2Actor.clear();
 204         member2Actor.dropMessagesToBehavior(RequestVote.class);
 205         member2Actor.dropMessagesToBehavior(AppendEntries.class);
 206
 207         member3Actor.clear();
 208         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
 209         member3Actor.expectMessageClass(AppendEntriesReply.class, 1);
 210
 211         member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 212
 213         member1Actor.waitForExpectedMessages(RequestVote.class);
 214         member2Actor.waitForExpectedMessages(RequestVote.class);
 215         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
 216
 217         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
 218         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
 219         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
 220
 221         // when member 3 switches to Leader it will immediately send out heartbeat AppendEntries to
 222         // the followers. Wait for AppendEntries to member 1 and its AppendEntriesReply. The
 223         // AppendEntries message to member 2 is dropped.
 224
 225         member1Actor.waitForExpectedMessages(AppendEntries.class);
 226         member2Actor.waitForExpectedMessages(AppendEntries.class);
 227         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 228
 229         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 230         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 231         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
 232
 233         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 234         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 235         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 236
 237         testLog.info("sendInitialElectionTimeoutToFollowerMember3 ending");
 238     }
 239
 240     private void sendInitialElectionTimeoutToFollowerMember2() {
 241         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 242
 243         // Send ElectionTimeout to member 2 to simulate no heartbeat from the Leader (member 1).
 244         // member 2 should switch to Candidate, start new term 2 and send out RequestVote messages.
 245         // member 1 will switch to Follower b/c its term is less than the member 2's RequestVote term, also it
 246         // won't send back a reply. member 3 will drop the message (ie won't forward it to its behavior) to
 247         // simulate loss of network connectivity between members 2 and 3.
 248
 249         member1Actor.expectMessageClass(RequestVote.class, 1);
 250
 251         member2Actor.expectBehaviorStateChange();
 252
 253         member3Actor.dropMessagesToBehavior(RequestVote.class);
 254
 255         member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 256
 257         member1Actor.waitForExpectedMessages(RequestVote.class);
 258         member3Actor.waitForExpectedMessages(RequestVote.class);
 259
 260         // Original leader member 1 should switch to Follower as the RequestVote term is greater than its
 261         // term. It won't send back a RequestVoteReply in this case.
 262
 263         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 264
 265         // member 2 should switch to Candidate since it didn't get a RequestVoteReply from the other 2 members.
 266
 267         member2Actor.waitForBehaviorStateChange();
 268         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 269
 270         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 271         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 272         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 273
 274         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 275     }
 276
 277     private void setupInitialMemberBehaviors() throws Exception {
 278         testLog.info("setupInitialMemberBehaviors starting");
 279
 280         // Create member 2's behavior initially as Follower
 281
 282         member2Context = newRaftActorContext("member2", member2ActorRef,
 283                 ImmutableMap.<String,String>builder().
 284                     put("member1", member1ActorRef.path().toString()).
 285                     put("member3", member3ActorRef.path().toString()).build());
 286
 287         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 288         member2Context.setConfigParams(member2ConfigParams);
 289
 290         member2Actor.behavior = new Follower(member2Context);
 291         member2Context.setCurrentBehavior(member2Actor.behavior);
 292
 293         // Create member 3's behavior initially as Follower
 294
 295         member3Context = newRaftActorContext("member3", member3ActorRef,
 296                 ImmutableMap.<String,String>builder().
 297                     put("member1", member1ActorRef.path().toString()).
 298                     put("member2", member2ActorRef.path().toString()).build());
 299
 300         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 301         member3Context.setConfigParams(member3ConfigParams);
 302
 303         member3Actor.behavior = new Follower(member3Context);
 304         member3Context.setCurrentBehavior(member3Actor.behavior);
 305
 306         // Create member 1's behavior initially as Leader
 307
 308         member1Context = newRaftActorContext("member1", member1ActorRef,
 309                 ImmutableMap.<String,String>builder().
 310                     put("member2", member2ActorRef.path().toString()).
 311                     put("member3", member3ActorRef.path().toString()).build());
 312
 313         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 314         member1Context.setConfigParams(member1ConfigParams);
 315
 316         initializeLeaderBehavior(member1Actor, member1Context, 2);
 317
 318         member2Actor.clear();
 319         member3Actor.clear();
 320
 321         testLog.info("setupInitialMemberBehaviors ending");
 322     }
 323 }