opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11
  12 import akka.actor.ActorRef;
  13 import com.google.common.collect.ImmutableMap;
  14 import org.junit.Test;
  15 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  16 import org.opendaylight.controller.cluster.raft.RaftState;
  17 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
  18 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  19 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  20 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  21
  22 /**
  23  * A leader election scenario test that delays various messages to behaviors to simulate network delays.
  24  *
  25  * @author Thomas Pantelis
  26  */
  27 public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  28
  29     @Test
  30     public void runTest() throws Exception {
  31         testLog.info("DelayedMessagesElectionScenarioTest starting");
  32
  33         setupInitialMemberBehaviors();
  34
  35         sendInitialElectionTimeoutToFollowerMember2();
  36
  37         forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3();
  38
  39         sendElectionTimeoutToFollowerMember3();
  40
  41         forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2();
  42
  43         testLog.info("DelayedMessagesElectionScenarioTest ending");
  44     }
  45
  46     private void forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2() throws Exception {
  47         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 starting");
  48
  49         // Now forward the original delayed RequestVoteReply from member 3 to member 2 that granted
  50         // the vote. Since member 2 is now a Follower, the RequestVoteReply should be ignored.
  51
  52         member2Actor.clearDropMessagesToBehavior();
  53         member2Actor.forwardCapturedMessageToBehavior(RequestVoteReply.class, member3ActorRef);
  54
  55         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
  56
  57         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  58         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  59         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  60
  61         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
  62         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
  63         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
  64
  65         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 ending");
  66     }
  67
  68     private void sendElectionTimeoutToFollowerMember3() throws Exception {
  69         testLog.info("sendElectionTimeoutToFollowerMember3 starting");
  70
  71         // Send ElectionTimeout to member 3 to simulate missing heartbeat from a Leader. member 3
  72         // should switch to Candidate and send out RequestVote messages. member 1 should grant the
  73         // vote and send a reply. After receiving the RequestVoteReply, member 3 should switch to leader.
  74
  75         member2Actor.expectBehaviorStateChange();
  76         member3Actor.clear();
  77         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
  78         member3Actor.expectMessageClass(AppendEntriesReply.class, 2);
  79
  80         member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
  81
  82         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
  83
  84         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
  85         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
  86         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
  87
  88         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  89
  90         // member 2 should've switched to Follower as member 3's RequestVote term (3) was greater
  91         // than member 2's term (2).
  92
  93         member2Actor.waitForBehaviorStateChange();
  94         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  95
  96         // The switch to leader should cause an immediate AppendEntries heartbeat from member 3.
  97
  98         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
  99
 100         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 101         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 102         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 103
 104         testLog.info("sendElectionTimeoutToFollowerMember3 ending");
 105     }
 106
 107     private void forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3() throws Exception {
 108         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 starting");
 109
 110         // At this point member 1 and 3 actors have captured the RequestVote messages. First
 111         // forward the RequestVote message to member 1's behavior. Since the RequestVote term
 112         // is greater than member 1's term and member 1 is a Leader, member 1 should switch to Follower
 113         // without replying to RequestVote and update its term to 2.
 114
 115         member1Actor.clearDropMessagesToBehavior();
 116         member1Actor.expectBehaviorStateChange();
 117         member1Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 118         member1Actor.waitForExpectedMessages(RequestVote.class);
 119
 120         member1Actor.waitForBehaviorStateChange();
 121         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 122
 123         // Now forward member 3's captured RequestVote message to its behavior. Since member 3 is
 124         // already a Follower, it should update its term to 2 and send a RequestVoteReply back to
 125         // member 2 granting the vote b/c the RequestVote's term, lastLogTerm, and lastLogIndex
 126         // should satisfy the criteria for granting the vote. However, we'll delay sending the
 127         // RequestVoteReply to member 2's behavior to simulate network latency.
 128
 129         member2Actor.dropMessagesToBehavior(RequestVoteReply.class);
 130
 131         member3Actor.clearDropMessagesToBehavior();
 132         member3Actor.expectMessageClass(RequestVote.class, 1);
 133         member3Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 134         member3Actor.waitForExpectedMessages(RequestVote.class);
 135         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 136
 137         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 138         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 139         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 140
 141         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 ending");
 142     }
 143
 144     private void sendInitialElectionTimeoutToFollowerMember2() {
 145         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 146
 147         // Send ElectionTimeout to member 2 to simulate missing heartbeat from the Leader. member 2
 148         // should switch to Candidate and send out RequestVote messages. Set member 1 and 3 actors
 149         // to capture RequestVote but not to forward to the behavior just yet as we want to
 150         // control the order of RequestVote messages to member 1 and 3.
 151
 152         member1Actor.dropMessagesToBehavior(RequestVote.class);
 153
 154         member2Actor.expectBehaviorStateChange();
 155
 156         member3Actor.dropMessagesToBehavior(RequestVote.class);
 157
 158         member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 159
 160         member1Actor.waitForExpectedMessages(RequestVote.class);
 161         member3Actor.waitForExpectedMessages(RequestVote.class);
 162
 163         member2Actor.waitForBehaviorStateChange();
 164         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 165
 166         assertEquals("member 1 election term", 1, member1Context.getTermInformation().getCurrentTerm());
 167         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 168         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 169
 170         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 171     }
 172
 173     private void setupInitialMemberBehaviors() throws Exception {
 174         testLog.info("setupInitialMemberBehaviors starting");
 175
 176         // Create member 2's behavior initially as Follower
 177
 178         member2Context = newRaftActorContext("member2", member2ActorRef,
 179                 ImmutableMap.<String,String>builder()
 180                     .put("member1", member1ActorRef.path().toString())
 181                     .put("member3", member3ActorRef.path().toString()).build());
 182
 183         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 184         member2Context.setConfigParams(member2ConfigParams);
 185
 186         member2Actor.behavior = new Follower(member2Context);
 187         member2Context.setCurrentBehavior(member2Actor.behavior);
 188
 189         // Create member 3's behavior initially as Follower
 190
 191         member3Context = newRaftActorContext("member3", member3ActorRef,
 192                 ImmutableMap.<String,String>builder()
 193                     .put("member1", member1ActorRef.path().toString())
 194                     .put("member2", member2ActorRef.path().toString()).build());
 195
 196         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 197         member3Context.setConfigParams(member3ConfigParams);
 198
 199         member3Actor.behavior = new Follower(member3Context);
 200         member3Context.setCurrentBehavior(member3Actor.behavior);
 201
 202         // Create member 1's behavior initially as Leader
 203
 204         member1Context = newRaftActorContext("member1", member1ActorRef,
 205                 ImmutableMap.<String,String>builder()
 206                     .put("member2", member2ActorRef.path().toString())
 207                     .put("member3", member3ActorRef.path().toString()).build());
 208
 209         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 210         member1Context.setConfigParams(member1ConfigParams);
 211
 212         initializeLeaderBehavior(member1Actor, member1Context, 2);
 213
 214         member2Actor.clear();
 215         member3Actor.clear();
 216
 217         testLog.info("setupInitialMemberBehaviors ending");
 218
 219     }
 220 }