opendaylight/md-sal/eos-dom-akka/src/test/java/org/opendaylight/controller/eos/akka/ThreeNodeReachabilityTest.java

   1 /*
   2  * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.eos.akka;
   9
  10 import static org.awaitility.Awaitility.await;
  11
  12 import akka.actor.testkit.typed.javadsl.ActorTestKit;
  13 import akka.cluster.Member;
  14 import akka.cluster.MemberStatus;
  15 import akka.cluster.typed.Cluster;
  16 import com.google.common.collect.ImmutableList;
  17 import java.time.Duration;
  18 import java.util.List;
  19 import org.junit.After;
  20 import org.junit.Before;
  21 import org.junit.Test;
  22 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
  23
  24 public class ThreeNodeReachabilityTest extends AbstractNativeEosTest {
  25     public static final DOMEntity ENTITY_1 = new DOMEntity("test-type", "entity-1");
  26     public static final DOMEntity ENTITY_2 = new DOMEntity("test-type-2", "entity-2");
  27
  28     private ClusterNode node1 = null;
  29     private ClusterNode node2 = null;
  30     private ClusterNode node3 = null;
  31
  32     @Before
  33     public void setUp() throws Exception {
  34         node1 = startupRemote(2550, List.of("member-1"), TWO_NODE_SEED_NODES);
  35         node2 = startupRemote(2551, List.of("member-2"), TWO_NODE_SEED_NODES);
  36
  37         // need to wait until all nodes are ready
  38         final Cluster cluster = Cluster.get(node2.getActorSystem());
  39         await().atMost(Duration.ofSeconds(20)).until(() -> {
  40             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
  41             if (members.size() != 2) {
  42                 return false;
  43             }
  44
  45             for (final Member member : members) {
  46                 if (!member.status().equals(MemberStatus.up())) {
  47                     return false;
  48                 }
  49             }
  50
  51             return true;
  52         });
  53     }
  54
  55     @After
  56     public void tearDown() {
  57         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
  58         ActorTestKit.shutdown(node2.getActorSystem(), Duration.ofSeconds(20));
  59
  60         if (node3 != null) {
  61             ActorTestKit.shutdown(node3.getActorSystem(), Duration.ofSeconds(20));
  62         }
  63     }
  64
  65     @Test
  66     public void testNodeLateStart() throws Exception {
  67         registerCandidates(node1, ENTITY_1, "member-1");
  68         registerCandidates(node2, ENTITY_1, "member-2");
  69
  70         registerCandidates(node2, ENTITY_2, "member-2");
  71         waitUntillOwnerPresent(node2, ENTITY_2);
  72         registerCandidates(node1, ENTITY_2, "member-1");
  73
  74         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
  75         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
  76
  77         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
  78         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
  79
  80         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
  81         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
  82
  83         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
  84         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
  85
  86         unregisterCandidates(node1, ENTITY_1, "member-1");
  87
  88         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, true);
  89         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
  90
  91         unregisterCandidates(node2, ENTITY_1, "member-2");
  92
  93         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, false);
  94         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, true);
  95
  96         startNode3();
  97
  98         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
  99         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
 100
 101         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
 102         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 103
 104         registerCandidates(node3, ENTITY_1, "member-3");
 105         waitUntillOwnerPresent(node3, ENTITY_1);
 106
 107         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 108         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
 109
 110         verifyListenerState(firstEntityListener3, ENTITY_1, true, true, false);
 111     }
 112
 113     @Test
 114     public void testReachabilityChangesDuringRuntime() throws Exception {
 115         startNode3();
 116
 117         registerCandidates(node2, ENTITY_1, "member-2");
 118         // we want singleton on node1 but owner on node2
 119         waitUntillOwnerPresent(node2, ENTITY_1);
 120
 121         registerCandidates(node1, ENTITY_1, "member-1");
 122         registerCandidates(node3, ENTITY_1, "member-3");
 123
 124         registerCandidates(node2, ENTITY_2, "member-2");
 125         waitUntillOwnerPresent(node2, ENTITY_2);
 126         registerCandidates(node1, ENTITY_2, "member-1");
 127
 128         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
 129         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
 130         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
 131
 132         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 133         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
 134         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 135
 136         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
 137         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
 138         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
 139
 140         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
 141         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
 142         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 143
 144         unreachableMember(node1, "member-2", DEFAULT_DATACENTER);
 145
 146         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
 147         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
 148         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 149
 150         verifyListenerState(secondEntityListener1, ENTITY_2, true, true, false);
 151         verifyListenerState(secondEntityListener2, ENTITY_2, true, false, true);
 152         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
 153
 154         unreachableMember(node1, "member-3", DEFAULT_DATACENTER);
 155
 156         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
 157         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
 158         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 159
 160         unregisterCandidates(node1, ENTITY_1, "member-1", DEFAULT_DATACENTER);
 161         unregisterCandidates(node1, ENTITY_2, "member-1", DEFAULT_DATACENTER);
 162
 163         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, true);
 164         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, false);
 165         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
 166
 167         verifyListenerState(secondEntityListener1, ENTITY_2, false, false, true);
 168         verifyListenerState(secondEntityListener2, ENTITY_2, false, false, false);
 169         verifyListenerState(secondEntityListener3, ENTITY_2, false, false, false);
 170
 171         reachableMember(node1, "member-2", DEFAULT_DATACENTER);
 172         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
 173         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
 174         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
 175     }
 176
 177     @Test
 178     public void testSingletonMoving() throws Exception {
 179         final MockEntityOwnershipListener listener1 = registerListener(node2, ENTITY_1);
 180         final MockEntityOwnershipListener listener2 = registerListener(node2, ENTITY_2);
 181         verifyNoNotifications(listener1);
 182         verifyNoNotifications(listener2);
 183
 184         registerCandidates(node1, ENTITY_1, "member-1");
 185         registerCandidates(node2, ENTITY_1, "member-2");
 186
 187         registerCandidates(node2, ENTITY_2, "member-2");
 188         waitUntillOwnerPresent(node2, ENTITY_2);
 189         registerCandidates(node1, ENTITY_2, "member-1");
 190         // end up with node1 - member-1, node2 - member-2 owners
 191         verifyListenerState(listener1, ENTITY_1, true, false, false);
 192         verifyListenerState(listener2, ENTITY_2, true, true, false);
 193
 194         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
 195
 196         verifyListenerState(listener1, ENTITY_1, true, true, false);
 197         verifyListenerState(listener2, ENTITY_2, true, true, false);
 198
 199         startNode3(2);
 200
 201         final MockEntityOwnershipListener listener3 = registerListener(node3, ENTITY_2);
 202         verifyListenerState(listener3, ENTITY_2, true, false, false);
 203
 204         node1 = startupRemote(2550, List.of("member-1"));
 205
 206         final Cluster cluster = Cluster.get(node2.getActorSystem());
 207         await().atMost(Duration.ofSeconds(20)).until(() -> {
 208             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
 209             if (members.size() != 3) {
 210                 return false;
 211             }
 212
 213             for (final Member member : members) {
 214                 if (!member.status().equals(MemberStatus.up())) {
 215                     return false;
 216                 }
 217             }
 218
 219             return true;
 220         });
 221
 222         final MockEntityOwnershipListener node1Listener = registerListener(node1, ENTITY_1);
 223         verifyListenerState(node1Listener, ENTITY_1, true, false, false);
 224     }
 225
 226     private void startNode3() throws Exception {
 227         startNode3(3);
 228     }
 229
 230     private void startNode3(final int membersPresent) throws Exception {
 231         node3 = startupRemote(2552, List.of("member-3"), THREE_NODE_SEED_NODES);
 232
 233         // need to wait until all nodes are ready
 234         final Cluster cluster = Cluster.get(node2.getActorSystem());
 235         await().atMost(Duration.ofSeconds(20)).until(() -> {
 236             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
 237             if (members.size() != membersPresent) {
 238                 return false;
 239             }
 240
 241             for (final Member member : members) {
 242                 if (!member.status().equals(MemberStatus.up())) {
 243                     return false;
 244                 }
 245             }
 246
 247             return true;
 248         });
 249     }
 250 }