opendaylight/md-sal/eos-dom-akka/src/test/java/org/opendaylight/controller/eos/akka/ThreeNodeReachabilityTest.java

   1 /*
   2  * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.eos.akka;
   9
  10 import static org.awaitility.Awaitility.await;
  11
  12 import akka.actor.testkit.typed.javadsl.ActorTestKit;
  13 import akka.cluster.Member;
  14 import akka.cluster.MemberStatus;
  15 import akka.cluster.typed.Cluster;
  16 import com.google.common.collect.ImmutableList;
  17 import java.time.Duration;
  18 import java.util.List;
  19 import org.junit.After;
  20 import org.junit.Before;
  21 import org.junit.Test;
  22 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
  23
  24 public class ThreeNodeReachabilityTest extends AbstractNativeEosTest {
  25     public static final DOMEntity ENTITY_1 = new DOMEntity("test-type", "entity-1");
  26     public static final DOMEntity ENTITY_2 = new DOMEntity("test-type-2", "entity-2");
  27
  28     private ClusterNode node1 = null;
  29     private ClusterNode node2 = null;
  30     private ClusterNode node3 = null;
  31
  32     @Before
  33     public void setUp() throws Exception {
  34         node1 = startupRemote(2550, List.of("member-1"), TWO_NODE_SEED_NODES);
  35         node2 = startupRemote(2551, List.of("member-2"), TWO_NODE_SEED_NODES);
  36
  37         // need to wait until all nodes are ready
  38         final Cluster cluster = Cluster.get(node2.getActorSystem());
  39         await().atMost(Duration.ofSeconds(20)).until(() -> {
  40             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
  41             if (members.size() != 2) {
  42                 return false;
  43             }
  44
  45             for (final Member member : members) {
  46                 if (!member.status().equals(MemberStatus.up())) {
  47                     return false;
  48                 }
  49             }
  50
  51             return true;
  52         });
  53     }
  54
  55     @After
  56     public void tearDown() {
  57         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
  58         ActorTestKit.shutdown(node2.getActorSystem(), Duration.ofSeconds(20));
  59
  60
  61         if (node3 != null) {
  62             ActorTestKit.shutdown(node3.getActorSystem(), Duration.ofSeconds(20));
  63         }
  64     }
  65
  66     @Test
  67     public void testNodeLateStart() throws Exception {
  68         registerCandidates(node1, ENTITY_1, "member-1");
  69         registerCandidates(node2, ENTITY_1, "member-2");
  70
  71         registerCandidates(node2, ENTITY_2, "member-2");
  72         waitUntillOwnerPresent(node2, ENTITY_2);
  73         registerCandidates(node1, ENTITY_2, "member-1");
  74
  75         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
  76         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
  77
  78         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
  79         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
  80
  81         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
  82         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
  83
  84         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
  85         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
  86
  87         unregisterCandidates(node1, ENTITY_1, "member-1");
  88
  89         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, true);
  90         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
  91
  92         unregisterCandidates(node2, ENTITY_1, "member-2");
  93
  94         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, false);
  95         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, true);
  96
  97         startNode3();
  98
  99         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
 100         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
 101
 102         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
 103         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 104
 105         registerCandidates(node3, ENTITY_1, "member-3");
 106         waitUntillOwnerPresent(node3, ENTITY_1);
 107
 108         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 109         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
 110
 111         verifyListenerState(firstEntityListener3, ENTITY_1, true, true, false);
 112     }
 113
 114     @Test
 115     public void testReachabilityChangesDuringRuntime() throws Exception {
 116         startNode3();
 117
 118         registerCandidates(node2, ENTITY_1, "member-2");
 119         // we want singleton on node1 but owner on node2
 120         waitUntillOwnerPresent(node2, ENTITY_1);
 121
 122         registerCandidates(node1, ENTITY_1, "member-1");
 123         registerCandidates(node3, ENTITY_1, "member-3");
 124
 125         registerCandidates(node2, ENTITY_2, "member-2");
 126         waitUntillOwnerPresent(node2, ENTITY_2);
 127         registerCandidates(node1, ENTITY_2, "member-1");
 128
 129         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
 130         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
 131         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
 132
 133         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 134         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
 135         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 136
 137         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
 138         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
 139         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
 140
 141         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
 142         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
 143         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 144
 145         unreachableMember(node1, "member-2");
 146
 147         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
 148         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
 149         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 150
 151         verifyListenerState(secondEntityListener1, ENTITY_2, true, true, false);
 152         verifyListenerState(secondEntityListener2, ENTITY_2, true, false, true);
 153         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 154
 155         unreachableMember(node1, "member-3");
 156
 157         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
 158         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
 159         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 160
 161         unregisterCandidates(node1, ENTITY_1, "member-1");
 162         unregisterCandidates(node1, ENTITY_2, "member-1");
 163
 164         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, true);
 165         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, false);
 166         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
 167
 168         verifyListenerState(secondEntityListener1, ENTITY_2, false, false, true);
 169         verifyListenerState(secondEntityListener2, ENTITY_2, false, false, false);
 170         verifyListenerState(secondEntityListener3, ENTITY_2, false, false, false);
 171
 172         reachableMember(node1, "member-2");
 173         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 174         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
 175         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 176     }
 177
 178     @Test
 179     public void testSingletonMoving() throws Exception {
 180         final MockEntityOwnershipListener listener1 = registerListener(node2, ENTITY_1);
 181         final MockEntityOwnershipListener listener2 = registerListener(node2, ENTITY_2);
 182         verifyNoNotifications(listener1);
 183         verifyNoNotifications(listener2);
 184
 185         registerCandidates(node1, ENTITY_1, "member-1");
 186         registerCandidates(node2, ENTITY_1, "member-2");
 187
 188         registerCandidates(node2, ENTITY_2, "member-2");
 189         waitUntillOwnerPresent(node2, ENTITY_2);
 190         registerCandidates(node1, ENTITY_2, "member-1");
 191         // end up with node1 - member-1, node2 - member-2 owners
 192         verifyListenerState(listener1, ENTITY_1, true, false, false);
 193         verifyListenerState(listener2, ENTITY_2, true, true, false);
 194
 195         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
 196
 197         verifyListenerState(listener1, ENTITY_1, true, true, false);
 198         verifyListenerState(listener2, ENTITY_2, true, true, false);
 199
 200         startNode3(2);
 201
 202         final MockEntityOwnershipListener listener3 = registerListener(node3, ENTITY_2);
 203         verifyListenerState(listener3, ENTITY_2, true, false, false);
 204
 205         node1 = startupRemote(2550, List.of("member-1"));
 206
 207         final Cluster cluster = Cluster.get(node2.getActorSystem());
 208         await().atMost(Duration.ofSeconds(20)).until(() -> {
 209             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
 210             if (members.size() != 3) {
 211                 return false;
 212             }
 213
 214             for (final Member member : members) {
 215                 if (!member.status().equals(MemberStatus.up())) {
 216                     return false;
 217                 }
 218             }
 219
 220             return true;
 221         });
 222
 223         final MockEntityOwnershipListener node1Listener = registerListener(node1, ENTITY_1);
 224         verifyListenerState(node1Listener, ENTITY_1, true, false, false);
 225     }
 226
 227     private void startNode3() throws Exception {
 228         startNode3(3);
 229     }
 230
 231     private void startNode3(final int membersPresent) throws Exception {
 232         node3 = startupRemote(2552, List.of("member-3"), THREE_NODE_SEED_NODES);
 233
 234         // need to wait until all nodes are ready
 235         final Cluster cluster = Cluster.get(node2.getActorSystem());
 236         await().atMost(Duration.ofSeconds(20)).until(() -> {
 237             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
 238             if (members.size() != membersPresent) {
 239                 return false;
 240             }
 241
 242             for (final Member member : members) {
 243                 if (!member.status().equals(MemberStatus.up())) {
 244                     return false;
 245                 }
 246             }
 247
 248             return true;
 249         });
 250     }
 251 }