opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/DelayedMessagesElectionScenarioTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.raft.behaviors;
   9
  10 import static org.junit.Assert.assertEquals;
  11
  12 import akka.actor.ActorRef;
  13 import com.google.common.collect.ImmutableMap;
  14 import org.junit.Test;
  15 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
  16 import org.opendaylight.controller.cluster.raft.RaftState;
  17 import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
  18 import org.opendaylight.controller.cluster.raft.messages.AppendEntries;
  19 import org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply;
  20 import org.opendaylight.controller.cluster.raft.messages.RequestVote;
  21 import org.opendaylight.controller.cluster.raft.messages.RequestVoteReply;
  22
  23 /**
  24  * A leader election scenario test that delays various messages to behaviors to simulate network delays.
  25  *
  26  * @author Thomas Pantelis
  27  */
  28 public class DelayedMessagesElectionScenarioTest extends AbstractLeaderElectionScenarioTest {
  29
  30     @Test
  31     public void runTest() {
  32         testLog.info("DelayedMessagesElectionScenarioTest starting");
  33
  34         setupInitialMemberBehaviors();
  35
  36         sendInitialElectionTimeoutToFollowerMember2();
  37
  38         forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3();
  39
  40         sendElectionTimeoutToFollowerMember3();
  41
  42         forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2();
  43
  44         testLog.info("DelayedMessagesElectionScenarioTest ending");
  45     }
  46
  47     private void forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2() {
  48         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 starting");
  49
  50         // Now forward the original delayed RequestVoteReply from member 3 to member 2 that granted
  51         // the vote. Since member 2 is now a Follower, the RequestVoteReply should be ignored.
  52
  53         member2Actor.clearDropMessagesToBehavior();
  54         member2Actor.forwardCapturedMessageToBehavior(RequestVoteReply.class, member3ActorRef);
  55
  56         member2Actor.waitForExpectedMessages(RequestVoteReply.class);
  57
  58         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
  59         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  60         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  61
  62         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
  63         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
  64         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
  65
  66         testLog.info("forwardDelayedRequestVoteReplyFromOriginalFollowerMember3ToMember2 ending");
  67     }
  68
  69     private void sendElectionTimeoutToFollowerMember3() {
  70         testLog.info("sendElectionTimeoutToFollowerMember3 starting");
  71
  72         // Send ElectionTimeout to member 3 to simulate missing heartbeat from a Leader. member 3
  73         // should switch to Candidate and send out RequestVote messages. member 1 should grant the
  74         // vote and send a reply. After receiving the RequestVoteReply, member 3 should switch to leader.
  75
  76         member2Actor.expectBehaviorStateChange();
  77         member3Actor.clear();
  78         member3Actor.expectMessageClass(RequestVoteReply.class, 1);
  79         member3Actor.expectMessageClass(AppendEntriesReply.class, 2);
  80
  81         member3ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
  82
  83         member3Actor.waitForExpectedMessages(RequestVoteReply.class);
  84
  85         RequestVoteReply requestVoteReply = member3Actor.getCapturedMessage(RequestVoteReply.class);
  86         assertEquals("getTerm", member3Context.getTermInformation().getCurrentTerm(), requestVoteReply.getTerm());
  87         assertEquals("isVoteGranted", true, requestVoteReply.isVoteGranted());
  88
  89         verifyBehaviorState("member 3", member3Actor, RaftState.Leader);
  90
  91         // member 2 should've switched to Follower as member 3's RequestVote term (3) was greater
  92         // than member 2's term (2).
  93
  94         member2Actor.waitForBehaviorStateChange();
  95         verifyBehaviorState("member 2", member2Actor, RaftState.Follower);
  96
  97         // The switch to leader should cause an immediate AppendEntries heartbeat from member 3.
  98
  99         member3Actor.waitForExpectedMessages(AppendEntriesReply.class);
 100
 101         assertEquals("member 1 election term", 3, member1Context.getTermInformation().getCurrentTerm());
 102         assertEquals("member 2 election term", 3, member2Context.getTermInformation().getCurrentTerm());
 103         assertEquals("member 3 election term", 3, member3Context.getTermInformation().getCurrentTerm());
 104
 105         testLog.info("sendElectionTimeoutToFollowerMember3 ending");
 106     }
 107
 108     private void forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3() {
 109         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 starting");
 110
 111         // At this point member 1 and 3 actors have captured the RequestVote messages. First
 112         // forward the RequestVote message to member 1's behavior. Since the RequestVote term
 113         // is greater than member 1's term and member 1 is a Leader, member 1 should switch to Follower
 114         // without replying to RequestVote and update its term to 2.
 115
 116         member1Actor.clearDropMessagesToBehavior();
 117         member1Actor.expectBehaviorStateChange();
 118         member1Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 119         member1Actor.waitForExpectedMessages(RequestVote.class);
 120
 121         member1Actor.waitForBehaviorStateChange();
 122         verifyBehaviorState("member 1", member1Actor, RaftState.Follower);
 123
 124         // Now forward member 3's captured RequestVote message to its behavior. Since member 3 is
 125         // already a Follower, it should update its term to 2 and send a RequestVoteReply back to
 126         // member 2 granting the vote b/c the RequestVote's term, lastLogTerm, and lastLogIndex
 127         // should satisfy the criteria for granting the vote. However, we'll delay sending the
 128         // RequestVoteReply to member 2's behavior to simulate network latency.
 129
 130         member2Actor.dropMessagesToBehavior(RequestVoteReply.class);
 131
 132         member3Actor.clearDropMessagesToBehavior();
 133         member3Actor.expectMessageClass(RequestVote.class, 1);
 134         member3Actor.forwardCapturedMessageToBehavior(RequestVote.class, member2ActorRef);
 135         member3Actor.waitForExpectedMessages(RequestVote.class);
 136         verifyBehaviorState("member 3", member3Actor, RaftState.Follower);
 137
 138         assertEquals("member 1 election term", 2, member1Context.getTermInformation().getCurrentTerm());
 139         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 140         assertEquals("member 3 election term", 2, member3Context.getTermInformation().getCurrentTerm());
 141
 142         testLog.info("forwardDelayedRequestVotesToLeaderMember1AndFollowerMember3 ending");
 143     }
 144
 145     private void sendInitialElectionTimeoutToFollowerMember2() {
 146         testLog.info("sendInitialElectionTimeoutToFollowerMember2 starting");
 147
 148         // Send ElectionTimeout to member 2 to simulate missing heartbeat from the Leader. member 2
 149         // should switch to Candidate and send out RequestVote messages. Set member 1 and 3 actors
 150         // to capture RequestVote but not to forward to the behavior just yet as we want to
 151         // control the order of RequestVote messages to member 1 and 3.
 152         member2Actor.expectBehaviorStateChange();
 153
 154         // member 1 and member 3 may reach consensus to consider leader's initial Noop entry as committed, hence
 155         // leader would elicit this information to member 2.
 156         // We do not want that, as member 2 would respond to that request either before it bumps or after it bumps its
 157         // term -- if it would see that message post-bump, it would leak term 2 back to member 1, hence leader would
 158         // know about it.
 159         member2Actor.dropMessagesToBehavior(AppendEntries.class);
 160
 161         member1Actor.dropMessagesToBehavior(RequestVote.class);
 162         member3Actor.dropMessagesToBehavior(RequestVote.class);
 163
 164         member2ActorRef.tell(TimeoutNow.INSTANCE, ActorRef.noSender());
 165         member1Actor.waitForExpectedMessages(RequestVote.class);
 166         member3Actor.waitForExpectedMessages(RequestVote.class);
 167
 168         member2Actor.waitForBehaviorStateChange();
 169         verifyBehaviorState("member 2", member2Actor, RaftState.Candidate);
 170
 171         assertEquals("member 1 election term", 1, member1Context.getTermInformation().getCurrentTerm());
 172         assertEquals("member 2 election term", 2, member2Context.getTermInformation().getCurrentTerm());
 173         assertEquals("member 3 election term", 1, member3Context.getTermInformation().getCurrentTerm());
 174
 175         testLog.info("sendInitialElectionTimeoutToFollowerMember2 ending");
 176     }
 177
 178     private void setupInitialMemberBehaviors() {
 179         testLog.info("setupInitialMemberBehaviors starting");
 180
 181         // Create member 2's behavior initially as Follower
 182
 183         member2Context = newRaftActorContext("member2", member2ActorRef,
 184                 ImmutableMap.<String,String>builder()
 185                     .put("member1", member1ActorRef.path().toString())
 186                     .put("member3", member3ActorRef.path().toString()).build());
 187
 188         DefaultConfigParamsImpl member2ConfigParams = newConfigParams();
 189         member2Context.setConfigParams(member2ConfigParams);
 190
 191         member2Actor.self().tell(new SetBehavior(new Follower(member2Context), member2Context),
 192                 ActorRef.noSender());
 193
 194         // Create member 3's behavior initially as Follower
 195
 196         member3Context = newRaftActorContext("member3", member3ActorRef,
 197                 ImmutableMap.<String,String>builder()
 198                     .put("member1", member1ActorRef.path().toString())
 199                     .put("member2", member2ActorRef.path().toString()).build());
 200
 201         DefaultConfigParamsImpl member3ConfigParams = newConfigParams();
 202         member3Context.setConfigParams(member3ConfigParams);
 203
 204         member3Actor.self().tell(new SetBehavior(new Follower(member3Context), member3Context),
 205                 ActorRef.noSender());
 206
 207         // Create member 1's behavior initially as Leader
 208
 209         member1Context = newRaftActorContext("member1", member1ActorRef,
 210                 ImmutableMap.<String,String>builder()
 211                     .put("member2", member2ActorRef.path().toString())
 212                     .put("member3", member3ActorRef.path().toString()).build());
 213
 214         DefaultConfigParamsImpl member1ConfigParams = newConfigParams();
 215         member1Context.setConfigParams(member1ConfigParams);
 216
 217         initializeLeaderBehavior(member1Actor, member1Context, 2);
 218
 219         member2Actor.clear();
 220         member3Actor.clear();
 221
 222         testLog.info("setupInitialMemberBehaviors ending");
 223     }
 224 }