8dded6dd8dd3d1af1efffdfa1d04df13d07d775e
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / IntegrationTestKit.java
1 /*
2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.datastore;
9
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertTrue;
13 import static org.junit.Assert.fail;
14
15 import akka.actor.ActorRef;
16 import akka.actor.ActorSystem;
17 import akka.cluster.Cluster;
18 import akka.cluster.ClusterEvent.CurrentClusterState;
19 import akka.cluster.Member;
20 import akka.cluster.MemberStatus;
21 import com.google.common.base.Stopwatch;
22 import com.google.common.collect.Sets;
23 import com.google.common.util.concurrent.ListenableFuture;
24 import com.google.common.util.concurrent.Uninterruptibles;
25 import java.lang.reflect.Constructor;
26 import java.util.Optional;
27 import java.util.Set;
28 import java.util.concurrent.Callable;
29 import java.util.concurrent.TimeUnit;
30 import java.util.function.Consumer;
31 import org.mockito.Mockito;
32 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
33 import org.opendaylight.controller.cluster.datastore.config.Configuration;
34 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
35 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
36 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
37 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
38 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
39 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
40 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
41 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
42 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
43 import org.opendaylight.mdsal.dom.spi.store.DOMStoreReadTransaction;
44 import org.opendaylight.mdsal.dom.spi.store.DOMStoreThreePhaseCommitCohort;
45 import org.opendaylight.mdsal.dom.spi.store.DOMStoreTransactionChain;
46 import org.opendaylight.mdsal.dom.spi.store.DOMStoreWriteTransaction;
47 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
48 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
49 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52 import scala.concurrent.Await;
53 import scala.concurrent.Future;
54 import scala.concurrent.duration.FiniteDuration;
55
56 public class IntegrationTestKit extends ShardTestKit {
57
58     private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
59
60     protected DatastoreContext.Builder datastoreContextBuilder;
61     protected DatastoreSnapshot restoreFromSnapshot;
62     private final int commitTimeout;
63
64     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
65         this(actorSystem, datastoreContextBuilder, 7);
66     }
67
68     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder,
69             final int commitTimeout) {
70         super(actorSystem);
71         this.datastoreContextBuilder = datastoreContextBuilder;
72         this.commitTimeout = commitTimeout;
73     }
74
75     public DatastoreContext.Builder getDatastoreContextBuilder() {
76         return datastoreContextBuilder;
77     }
78
79     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
80                                                           final boolean waitUntilLeader,
81                                                           final SchemaContext schemaContext) throws Exception {
82         return setupDistributedDataStore(typeName, moduleShardsConfig, "modules.conf", waitUntilLeader, schemaContext);
83     }
84
85     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
86                                                           final String modulesConfig,
87                                                           final boolean waitUntilLeader,
88                                                           final SchemaContext schemaContext,
89                                                           final String... shardNames) throws Exception {
90         return (DistributedDataStore) setupAbstractDataStore(DistributedDataStore.class, typeName, moduleShardsConfig,
91                 modulesConfig, waitUntilLeader, schemaContext, shardNames);
92     }
93
94     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
95                                                     final String typeName, final String... shardNames)
96             throws Exception {
97         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", true,
98                 SchemaContextHelper.full(), shardNames);
99     }
100
101     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
102                                                     final String typeName, final boolean waitUntilLeader,
103                                                     final String... shardNames) throws Exception {
104         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
105                 SchemaContextHelper.full(), shardNames);
106     }
107
108     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
109                                                     final String typeName, final String moduleShardsConfig,
110                                                     final boolean waitUntilLeader, final String... shardNames)
111             throws Exception {
112         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
113                 SchemaContextHelper.full(), shardNames);
114     }
115
116     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
117                                                     final String typeName, final String moduleShardsConfig,
118                                                     final boolean waitUntilLeader,
119                                                     final SchemaContext schemaContext,
120                                                     final String... shardNames) throws Exception {
121         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
122                 schemaContext, shardNames);
123     }
124
125     private AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
126                                                      final String typeName, final String moduleShardsConfig,
127                                                      final String modulesConfig, final boolean waitUntilLeader,
128                                                      final SchemaContext schemaContext, final String... shardNames)
129             throws Exception {
130         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
131         final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
132
133         setDataStoreName(typeName);
134
135         final DatastoreContext datastoreContext = datastoreContextBuilder.build();
136         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
137         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
138         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
139
140         final Constructor<? extends AbstractDataStore> constructor = implementation.getDeclaredConstructor(
141                 ActorSystem.class, ClusterWrapper.class, Configuration.class,
142                 DatastoreContextFactory.class, DatastoreSnapshot.class);
143
144         final AbstractDataStore dataStore = constructor.newInstance(getSystem(), cluster, config, mockContextFactory,
145             restoreFromSnapshot);
146
147         dataStore.onGlobalContextUpdated(schemaContext);
148
149         if (waitUntilLeader) {
150             waitUntilLeader(dataStore.getActorUtils(), shardNames);
151         }
152
153         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
154         return dataStore;
155     }
156
157     private void setDataStoreName(final String typeName) {
158         if ("config".equals(typeName)) {
159             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.CONFIGURATION);
160         } else if ("operational".equals(typeName)) {
161             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.OPERATIONAL);
162         } else {
163             datastoreContextBuilder.dataStoreName(typeName);
164         }
165     }
166
167     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
168                                                                        final SchemaContext schemaContext) {
169         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
170         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
171
172         setDataStoreName(typeName);
173
174         final DatastoreContext datastoreContext = getDatastoreContextBuilder().build();
175
176         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
177         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
178         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
179
180         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
181                 configuration, mockContextFactory, restoreFromSnapshot);
182
183         dataStore.onGlobalContextUpdated(schemaContext);
184
185         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
186         return dataStore;
187     }
188
189     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
190                                                                        final SchemaContext schemaContext,
191                                                                        final LogicalDatastoreType storeType) {
192         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
193         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
194
195         setDataStoreName(typeName);
196
197         final DatastoreContext datastoreContext =
198                 getDatastoreContextBuilder().logicalStoreType(storeType).build();
199
200         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
201         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
202         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
203
204         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
205                 configuration, mockContextFactory, restoreFromSnapshot);
206
207         dataStore.onGlobalContextUpdated(schemaContext);
208
209         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
210         return dataStore;
211     }
212
213     public void waitUntilLeader(final ActorUtils actorUtils, final String... shardNames) {
214         for (String shardName: shardNames) {
215             ActorRef shard = findLocalShard(actorUtils, shardName);
216
217             assertNotNull("Shard was not created for " + shardName, shard);
218
219             waitUntilLeader(shard);
220         }
221     }
222
223     public void waitUntilNoLeader(final ActorUtils actorUtils, final String... shardNames) {
224         for (String shardName: shardNames) {
225             ActorRef shard = findLocalShard(actorUtils, shardName);
226             assertNotNull("No local shard found for " + shardName, shard);
227
228             waitUntilNoLeader(shard);
229         }
230     }
231
232     public void waitForMembersUp(final String... otherMembers) {
233         Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
234         Stopwatch sw = Stopwatch.createStarted();
235         while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
236             CurrentClusterState state = Cluster.get(getSystem()).state();
237             for (Member m: state.getMembers()) {
238                 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
239                         && otherMembersSet.isEmpty()) {
240                     return;
241                 }
242             }
243
244             Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
245         }
246
247         fail("Member(s) " + otherMembersSet + " are not Up");
248     }
249
250     public static ActorRef findLocalShard(final ActorUtils actorUtils, final String shardName) {
251         ActorRef shard = null;
252         for (int i = 0; i < 20 * 5 && shard == null; i++) {
253             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
254             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
255             if (shardReply.isPresent()) {
256                 shard = shardReply.get();
257             }
258         }
259         return shard;
260     }
261
262     public static void waitUntilShardIsDown(final ActorUtils actorUtils, final String shardName) {
263         for (int i = 0; i < 20 * 5 ; i++) {
264             LOG.debug("Waiting for shard down {}", shardName);
265             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
266             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
267             if (!shardReply.isPresent()) {
268                 return;
269             }
270         }
271
272         throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
273     }
274
275     public static void verifyShardStats(final AbstractDataStore datastore, final String shardName,
276             final ShardStatsVerifier verifier) throws Exception {
277         ActorUtils actorUtils = datastore.getActorUtils();
278
279         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
280         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
281
282         AssertionError lastError = null;
283         Stopwatch sw = Stopwatch.createStarted();
284         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
285             ShardStats shardStats = (ShardStats)actorUtils
286                     .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
287
288             try {
289                 verifier.verify(shardStats);
290                 return;
291             } catch (AssertionError e) {
292                 lastError = e;
293                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
294             }
295         }
296
297         throw lastError;
298     }
299
300     public static void verifyShardState(final AbstractDataStore datastore, final String shardName,
301             final Consumer<OnDemandShardState> verifier) throws Exception {
302         ActorUtils actorUtils = datastore.getActorUtils();
303
304         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
305         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
306
307         AssertionError lastError = null;
308         Stopwatch sw = Stopwatch.createStarted();
309         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
310             OnDemandShardState shardState = (OnDemandShardState)actorUtils
311                     .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
312
313             try {
314                 verifier.accept(shardState);
315                 return;
316             } catch (AssertionError e) {
317                 lastError = e;
318                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
319             }
320         }
321
322         throw lastError;
323     }
324
325     void testWriteTransaction(final AbstractDataStore dataStore, final YangInstanceIdentifier nodePath,
326             final NormalizedNode<?, ?> nodeToWrite) throws Exception {
327
328         // 1. Create a write-only Tx
329
330         DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
331         assertNotNull("newWriteOnlyTransaction returned null", writeTx);
332
333         // 2. Write some data
334
335         writeTx.write(nodePath, nodeToWrite);
336
337         // 3. Ready the Tx for commit
338
339         DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
340
341         // 4. Commit the Tx
342
343         doCommit(cohort);
344
345         // 5. Verify the data in the store
346
347         DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
348
349         Optional<NormalizedNode<?, ?>> optional = readTx.read(nodePath).get(5, TimeUnit.SECONDS);
350         assertTrue("isPresent", optional.isPresent());
351         assertEquals("Data node", nodeToWrite, optional.get());
352     }
353
354     public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
355         Boolean canCommit = cohort.canCommit().get(commitTimeout, TimeUnit.SECONDS);
356         assertEquals("canCommit", Boolean.TRUE, canCommit);
357         cohort.preCommit().get(5, TimeUnit.SECONDS);
358         cohort.commit().get(5, TimeUnit.SECONDS);
359     }
360
361     void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
362             throws Exception {
363         Boolean canCommit = canCommitFuture.get(commitTimeout, TimeUnit.SECONDS);
364         assertEquals("canCommit", Boolean.TRUE, canCommit);
365         cohort.preCommit().get(5, TimeUnit.SECONDS);
366         cohort.commit().get(5, TimeUnit.SECONDS);
367     }
368
369     @SuppressWarnings("checkstyle:IllegalCatch")
370     void assertExceptionOnCall(final Callable<Void> callable, final Class<? extends Exception> expType) {
371         try {
372             callable.call();
373             fail("Expected " + expType.getSimpleName());
374         } catch (Exception e) {
375             assertEquals("Exception type", expType, e.getClass());
376         }
377     }
378
379     void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
380             final Class<? extends Exception> expType) {
381         assertExceptionOnCall(() -> {
382             txChain.newWriteOnlyTransaction();
383             return null;
384         }, expType);
385
386         assertExceptionOnCall(() -> {
387             txChain.newReadWriteTransaction();
388             return null;
389         }, expType);
390
391         assertExceptionOnCall(() -> {
392             txChain.newReadOnlyTransaction();
393             return null;
394         }, expType);
395     }
396
397     public interface ShardStatsVerifier {
398         void verify(ShardStats stats);
399     }
400 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.