opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import akka.actor.ActorRef;
  12 import com.google.common.collect.ImmutableMap;
  13 import org.junit.Test;
  14 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  15 import org.opendaylight.controller.cluster.raft.RaftState;
  16 import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout;
  17 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  18 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  19 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  20
  21 /**
  22  * A leader election scenario test that delays various messages to behaviors to simulate network delays.
  23  *
  24  * @author Thomas Pantelis
  25  */
  26 public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  27
  28     @Test
  29     public void runTest() throws Exception {
  30         testLog.info("DelayedMessagesElectionScenarioTest starting");
  31
  32         setupInitialMemberBehaviors();
  33
  34         sendInitialElectionTimeoutToFollowerMember2();
  35
  36         forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3();
  37
  38         sendElectionTimeoutToFollowerMember3();
  39
  40         forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2();
  41
  42         testLog.info("DelayedMessagesElectionScenarioTest ending");
  43     }
  44
  45     private void forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2() throws Exception {
  46         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 starting");
  47
  48         // Now forward the original delayed RequestVoteReply from member 3 to member 2 that granted
  49         // the vote. Since member 2 is now a Follower, the RequestVoteReply should be ignored.
  50
  51         member2Actor.clearDropMessagesToBehavior();
  52         member2Actor.forwardCapturedMessageToBehavior(RequestVoteReply.class, member3ActorRef);
  53
  54         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
  55
  56         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  57         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  58         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  59
  60         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
  61         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
  62         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
  63
  64         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 ending");
  65     }
  66
  67     private void sendElectionTimeoutToFollowerMember3() throws Exception {
  68         testLog.info("sendElectionTimeoutToFollowerMember3 starting");
  69
  70         // Send ElectionTimeout to member 3 to simulate missing heartbeat from a Leader. member 3
  71         // should switch to Candidate and send out RequestVote messages. member 1 should grant the
  72         // vote and send a reply. After receiving the RequestVoteReply, member 3 should switch to leader.
  73
  74         member2Actor.expectBehaviorStateChange();
  75         member3Actor.clear();
  76         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
  77         member3Actor.expectMessageClass(AppendEntriesReply.class, 2);
  78
  79         member3ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
  80
  81         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
  82
  83         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
  84         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
  85         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
  86
  87         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  88
  89         // member 2 should've switched to Follower as member 3's RequestVote term (3) was greater
  90         // than member 2's term (2).
  91
  92         member2Actor.waitForBehaviorStateChange();
  93         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  94
  95         // The switch to leader should cause an immediate AppendEntries heartbeat from member 3.
  96
  97         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
  98
  99         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 100         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 101         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 102
 103         testLog.info("sendElectionTimeoutToFollowerMember3 ending");
 104     }
 105
 106     private void forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3() throws Exception {
 107         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 starting");
 108
 109         // At this point member 1 and 3 actors have captured the RequestVote messages. First
 110         // forward the RequestVote message to member 1's behavior. Since the RequestVote term
 111         // is greater than member 1's term and member 1 is a Leader, member 1 should switch to Follower
 112         // without replying to RequestVote and update its term to 2.
 113
 114         member1Actor.clearDropMessagesToBehavior();
 115         member1Actor.expectBehaviorStateChange();
 116         member1Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 117         member1Actor.waitForExpectedMessages(RequestVote.class);
 118
 119         member1Actor.waitForBehaviorStateChange();
 120         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 121
 122         // Now forward member 3's captured RequestVote message to its behavior. Since member 3 is
 123         // already a Follower, it should update its term to 2 and send a RequestVoteReply back to
 124         // member 2 granting the vote b/c the RequestVote's term, lastLogTerm, and lastLogIndex
 125         // should satisfy the criteria for granting the vote. However, we'll delay sending the
 126         // RequestVoteReply to member 2's behavior to simulate network latency.
 127
 128         member2Actor.dropMessagesToBehavior(RequestVoteReply.class);
 129
 130         member3Actor.clearDropMessagesToBehavior();
 131         member3Actor.expectMessageClass(RequestVote.class, 1);
 132         member3Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 133         member3Actor.waitForExpectedMessages(RequestVote.class);
 134         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 135
 136         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 137         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 138         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 139
 140         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 ending");
 141     }
 142
 143     private void sendInitialElectionTimeoutToFollowerMember2() {
 144         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 145
 146         // Send ElectionTimeout to member 2 to simulate missing heartbeat from the Leader. member 2
 147         // should switch to Candidate and send out RequestVote messages. Set member 1 and 3 actors
 148         // to capture RequestVote but not to forward to the behavior just yet as we want to
 149         // control the order of RequestVote messages to member 1 and 3.
 150
 151         member1Actor.dropMessagesToBehavior(RequestVote.class);
 152
 153         member2Actor.expectBehaviorStateChange();
 154
 155         member3Actor.dropMessagesToBehavior(RequestVote.class);
 156
 157         member2ActorRef.tell(ElectionTimeout.INSTANCE, ActorRef.noSender());
 158
 159         member1Actor.waitForExpectedMessages(RequestVote.class);
 160         member3Actor.waitForExpectedMessages(RequestVote.class);
 161
 162         member2Actor.waitForBehaviorStateChange();
 163         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 164
 165         assertEquals("member 1 election term", 1, member1Context.getTermInformation().getCurrentTerm());
 166         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 167         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 168
 169         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 170     }
 171
 172     private void setupInitialMemberBehaviors() throws Exception {
 173         testLog.info("setupInitialMemberBehaviors starting");
 174
 175         // Create member 2's behavior initially as Follower
 176
 177         member2Context = newRaftActorContext("member2", member2ActorRef,
 178                 ImmutableMap.<String,String>builder().
 179                     put("member1", member1ActorRef.path().toString()).
 180                     put("member3", member3ActorRef.path().toString()).build());
 181
 182         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 183         member2Context.setConfigParams(member2ConfigParams);
 184
 185         member2Actor.behavior = new Follower(member2Context);
 186
 187         // Create member 3's behavior initially as Follower
 188
 189         member3Context = newRaftActorContext("member3", member3ActorRef,
 190                 ImmutableMap.<String,String>builder().
 191                     put("member1", member1ActorRef.path().toString()).
 192                     put("member2", member2ActorRef.path().toString()).build());
 193
 194         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 195         member3Context.setConfigParams(member3ConfigParams);
 196
 197         member3Actor.behavior = new Follower(member3Context);
 198
 199         // Create member 1's behavior initially as Leader
 200
 201         member1Context = newRaftActorContext("member1", member1ActorRef,
 202                 ImmutableMap.<String,String>builder().
 203                     put("member2", member2ActorRef.path().toString()).
 204                     put("member3", member3ActorRef.path().toString()).build());
 205
 206         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 207         member1Context.setConfigParams(member1ConfigParams);
 208
 209         initializeLeaderBehavior(member1Actor, member1Context, 2);
 210
 211         member2Actor.clear();
 212         member3Actor.clear();
 213
 214         testLog.info("setupInitialMemberBehaviors ending");
 215
 216     }
 217 }