Change RTE to an ISE
[controller.git] / opendaylight / md-sal / eos-dom-akka / src / test / java / org / opendaylight / controller / eos / akka / ThreeNodeReachabilityTest.java
1 /*
2  * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.eos.akka;
9
10 import static org.awaitility.Awaitility.await;
11
12 import akka.actor.testkit.typed.javadsl.ActorTestKit;
13 import akka.cluster.Member;
14 import akka.cluster.MemberStatus;
15 import akka.cluster.typed.Cluster;
16 import com.google.common.collect.ImmutableList;
17 import java.time.Duration;
18 import java.util.List;
19 import org.junit.After;
20 import org.junit.Before;
21 import org.junit.Test;
22 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
23
24 public class ThreeNodeReachabilityTest extends AbstractNativeEosTest {
25     public static final DOMEntity ENTITY_1 = new DOMEntity("test-type", "entity-1");
26     public static final DOMEntity ENTITY_2 = new DOMEntity("test-type-2", "entity-2");
27
28     private ClusterNode node1 = null;
29     private ClusterNode node2 = null;
30     private ClusterNode node3 = null;
31
32     @Before
33     public void setUp() throws Exception {
34         node1 = startupRemote(2550, List.of("member-1"), TWO_NODE_SEED_NODES);
35         node2 = startupRemote(2551, List.of("member-2"), TWO_NODE_SEED_NODES);
36
37         // need to wait until all nodes are ready
38         final Cluster cluster = Cluster.get(node2.getActorSystem());
39         await().atMost(Duration.ofSeconds(20)).until(() -> {
40             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
41             if (members.size() != 2) {
42                 return false;
43             }
44
45             for (final Member member : members) {
46                 if (!member.status().equals(MemberStatus.up())) {
47                     return false;
48                 }
49             }
50
51             return true;
52         });
53     }
54
55     @After
56     public void tearDown() {
57         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
58         ActorTestKit.shutdown(node2.getActorSystem(), Duration.ofSeconds(20));
59
60         if (node3 != null) {
61             ActorTestKit.shutdown(node3.getActorSystem(), Duration.ofSeconds(20));
62         }
63     }
64
65     @Test
66     public void testNodeLateStart() throws Exception {
67         registerCandidates(node1, ENTITY_1, "member-1");
68         registerCandidates(node2, ENTITY_1, "member-2");
69
70         registerCandidates(node2, ENTITY_2, "member-2");
71         waitUntillOwnerPresent(node2, ENTITY_2);
72         registerCandidates(node1, ENTITY_2, "member-1");
73
74         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
75         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
76
77         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
78         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
79
80         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
81         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
82
83         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
84         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
85
86         unregisterCandidates(node1, ENTITY_1, "member-1");
87
88         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, true);
89         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
90
91         unregisterCandidates(node2, ENTITY_1, "member-2");
92
93         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, false);
94         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, true);
95
96         startNode3();
97
98         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
99         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
100
101         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
102         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
103
104         registerCandidates(node3, ENTITY_1, "member-3");
105         waitUntillOwnerPresent(node3, ENTITY_1);
106
107         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
108         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, false);
109
110         verifyListenerState(firstEntityListener3, ENTITY_1, true, true, false);
111     }
112
113     @Test
114     public void testReachabilityChangesDuringRuntime() throws Exception {
115         startNode3();
116
117         registerCandidates(node2, ENTITY_1, "member-2");
118         // we want singleton on node1 but owner on node2
119         waitUntillOwnerPresent(node2, ENTITY_1);
120
121         registerCandidates(node1, ENTITY_1, "member-1");
122         registerCandidates(node3, ENTITY_1, "member-3");
123
124         registerCandidates(node2, ENTITY_2, "member-2");
125         waitUntillOwnerPresent(node2, ENTITY_2);
126         registerCandidates(node1, ENTITY_2, "member-1");
127
128         final MockEntityOwnershipListener firstEntityListener1 = registerListener(node1, ENTITY_1);
129         final MockEntityOwnershipListener firstEntityListener2 = registerListener(node2, ENTITY_1);
130         final MockEntityOwnershipListener firstEntityListener3 = registerListener(node3, ENTITY_1);
131
132         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
133         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
134         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
135
136         final MockEntityOwnershipListener secondEntityListener1 = registerListener(node1, ENTITY_2);
137         final MockEntityOwnershipListener secondEntityListener2 = registerListener(node2, ENTITY_2);
138         final MockEntityOwnershipListener secondEntityListener3 = registerListener(node3, ENTITY_2);
139
140         verifyListenerState(secondEntityListener1, ENTITY_2, true, false, false);
141         verifyListenerState(secondEntityListener2, ENTITY_2, true, true, false);
142         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
143
144         unreachableMember(node1, "member-2", DEFAULT_DATACENTER);
145
146         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
147         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
148         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
149
150         verifyListenerState(secondEntityListener1, ENTITY_2, true, true, false);
151         verifyListenerState(secondEntityListener2, ENTITY_2, true, false, true);
152         verifyListenerState(secondEntityListener3, ENTITY_2, true, false, false);
153
154         unreachableMember(node1, "member-3", DEFAULT_DATACENTER);
155
156         verifyListenerState(firstEntityListener1, ENTITY_1, true, true, false);
157         verifyListenerState(firstEntityListener2, ENTITY_1, true, false, true);
158         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
159
160         unregisterCandidates(node1, ENTITY_1, "member-1", DEFAULT_DATACENTER);
161         unregisterCandidates(node1, ENTITY_2, "member-1", DEFAULT_DATACENTER);
162
163         verifyListenerState(firstEntityListener1, ENTITY_1, false, false, true);
164         verifyListenerState(firstEntityListener2, ENTITY_1, false, false, false);
165         verifyListenerState(firstEntityListener3, ENTITY_1, false, false, false);
166
167         verifyListenerState(secondEntityListener1, ENTITY_2, false, false, true);
168         verifyListenerState(secondEntityListener2, ENTITY_2, false, false, false);
169         verifyListenerState(secondEntityListener3, ENTITY_2, false, false, false);
170
171         reachableMember(node1, "member-2", DEFAULT_DATACENTER);
172         verifyListenerState(firstEntityListener1, ENTITY_1, true, false, false);
173         verifyListenerState(firstEntityListener2, ENTITY_1, true, true, false);
174         verifyListenerState(firstEntityListener3, ENTITY_1, true, false, false);
175     }
176
177     @Test
178     public void testSingletonMoving() throws Exception {
179         final MockEntityOwnershipListener listener1 = registerListener(node2, ENTITY_1);
180         final MockEntityOwnershipListener listener2 = registerListener(node2, ENTITY_2);
181         verifyNoNotifications(listener1);
182         verifyNoNotifications(listener2);
183
184         registerCandidates(node1, ENTITY_1, "member-1");
185         registerCandidates(node2, ENTITY_1, "member-2");
186
187         registerCandidates(node2, ENTITY_2, "member-2");
188         waitUntillOwnerPresent(node2, ENTITY_2);
189         registerCandidates(node1, ENTITY_2, "member-1");
190         // end up with node1 - member-1, node2 - member-2 owners
191         verifyListenerState(listener1, ENTITY_1, true, false, false);
192         verifyListenerState(listener2, ENTITY_2, true, true, false);
193
194         ActorTestKit.shutdown(node1.getActorSystem(), Duration.ofSeconds(20));
195
196         verifyListenerState(listener1, ENTITY_1, true, true, false);
197         verifyListenerState(listener2, ENTITY_2, true, true, false);
198
199         startNode3(2);
200
201         final MockEntityOwnershipListener listener3 = registerListener(node3, ENTITY_2);
202         verifyListenerState(listener3, ENTITY_2, true, false, false);
203
204         node1 = startupRemote(2550, List.of("member-1"));
205
206         final Cluster cluster = Cluster.get(node2.getActorSystem());
207         await().atMost(Duration.ofSeconds(20)).until(() -> {
208             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
209             if (members.size() != 3) {
210                 return false;
211             }
212
213             for (final Member member : members) {
214                 if (!member.status().equals(MemberStatus.up())) {
215                     return false;
216                 }
217             }
218
219             return true;
220         });
221
222         final MockEntityOwnershipListener node1Listener = registerListener(node1, ENTITY_1);
223         verifyListenerState(node1Listener, ENTITY_1, true, false, false);
224     }
225
226     @Test
227     public void testOwnerNotReassignedWhenOnlyCandidate() throws Exception {
228         startNode3();
229         final MockEntityOwnershipListener listener1 = registerListener(node1, ENTITY_1);
230         final MockEntityOwnershipListener listener2 = registerListener(node2, ENTITY_1);
231         verifyNoNotifications(listener1);
232         verifyNoNotifications(listener2);
233
234         registerCandidates(node3, ENTITY_1, "member-3");
235         waitUntillOwnerPresent(node1, ENTITY_1);
236
237         MockEntityOwnershipListener listener3 = registerListener(node3, ENTITY_1);
238         verifyListenerState(listener1, ENTITY_1, true, false, false);
239         verifyListenerState(listener3, ENTITY_1, true, true, false);
240
241         ActorTestKit.shutdown(node3.getActorSystem(), Duration.ofSeconds(20));
242
243         verifyListenerState(listener1, ENTITY_1, true, false, false);
244         verifyListenerState(listener2, ENTITY_1, true, false, false);
245
246         startNode3();
247         verifyListenerState(listener1, ENTITY_1, false, false, false);
248
249         listener3 = registerListener(node3, ENTITY_1);
250         verifyListenerState(listener3, ENTITY_1, false, false, false);
251
252         registerCandidates(node1, ENTITY_1, "member-1");
253
254         verifyListenerState(listener1, ENTITY_1, true, true, false);
255         verifyListenerState(listener3, ENTITY_1, true, false, false);
256
257     }
258
259     private void startNode3() throws Exception {
260         startNode3(3);
261     }
262
263     private void startNode3(final int membersPresent) throws Exception {
264         node3 = startupRemote(2552, List.of("member-3"), THREE_NODE_SEED_NODES);
265
266         // need to wait until all nodes are ready
267         final Cluster cluster = Cluster.get(node2.getActorSystem());
268         await().atMost(Duration.ofSeconds(30)).until(() -> {
269             final List<Member> members = ImmutableList.copyOf(cluster.state().getMembers());
270             if (members.size() != membersPresent) {
271                 return false;
272             }
273
274             for (final Member member : members) {
275                 if (!member.status().equals(MemberStatus.up())) {
276                     return false;
277                 }
278             }
279
280             return true;
281         });
282     }
283 }