opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.junit.Assert.assertThrows;
  13 import static org.junit.Assert.fail;
  14 import static org.mockito.ArgumentMatchers.anyString;
  15 import static org.mockito.Mockito.doReturn;
  16 import static org.mockito.Mockito.mock;
  17
  18 import akka.actor.ActorRef;
  19 import akka.actor.ActorSystem;
  20 import akka.cluster.Cluster;
  21 import akka.cluster.ClusterEvent.CurrentClusterState;
  22 import akka.cluster.Member;
  23 import akka.cluster.MemberStatus;
  24 import com.google.common.base.Stopwatch;
  25 import com.google.common.collect.Sets;
  26 import com.google.common.util.concurrent.ListenableFuture;
  27 import com.google.common.util.concurrent.Uninterruptibles;
  28 import java.lang.reflect.Constructor;
  29 import java.util.Optional;
  30 import java.util.Set;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.function.Consumer;
  33 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  34 import org.opendaylight.controller.cluster.datastore.config.Configuration;
  35 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  36 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
  37 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  38 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
  39 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
  40 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  41 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  42 import org.opendaylight.mdsal.dom.spi.store.DOMStoreReadTransaction;
  43 import org.opendaylight.mdsal.dom.spi.store.DOMStoreThreePhaseCommitCohort;
  44 import org.opendaylight.mdsal.dom.spi.store.DOMStoreTransactionChain;
  45 import org.opendaylight.mdsal.dom.spi.store.DOMStoreWriteTransaction;
  46 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  47 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  48 import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51 import scala.concurrent.Await;
  52 import scala.concurrent.Future;
  53 import scala.concurrent.duration.FiniteDuration;
  54
  55 public class IntegrationTestKit extends ShardTestKit {
  56
  57     private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
  58
  59     protected DatastoreContext.Builder datastoreContextBuilder;
  60     protected DatastoreSnapshot restoreFromSnapshot;
  61     private final int commitTimeout;
  62
  63     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
  64         this(actorSystem, datastoreContextBuilder, 7);
  65     }
  66
  67     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder,
  68             final int commitTimeout) {
  69         super(actorSystem);
  70         this.datastoreContextBuilder = datastoreContextBuilder;
  71         this.commitTimeout = commitTimeout;
  72     }
  73
  74     public DatastoreContext.Builder getDatastoreContextBuilder() {
  75         return datastoreContextBuilder;
  76     }
  77
  78     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  79                                                     final String typeName, final String... shardNames)
  80             throws Exception {
  81         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", true,
  82                 SchemaContextHelper.full(), shardNames);
  83     }
  84
  85     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  86                                                     final String typeName, final boolean waitUntilLeader,
  87                                                     final String... shardNames) throws Exception {
  88         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
  89                 SchemaContextHelper.full(), shardNames);
  90     }
  91
  92     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  93                                                     final String typeName, final String moduleShardsConfig,
  94                                                     final boolean waitUntilLeader, final String... shardNames)
  95             throws Exception {
  96         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
  97                 SchemaContextHelper.full(), shardNames);
  98     }
  99
 100     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 101                                                     final String typeName, final String moduleShardsConfig,
 102                                                     final boolean waitUntilLeader,
 103                                                     final EffectiveModelContext schemaContext,
 104                                                     final String... shardNames) throws Exception {
 105         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
 106                 schemaContext, shardNames);
 107     }
 108
 109     private AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 110                                                      final String typeName, final String moduleShardsConfig,
 111                                                      final String modulesConfig, final boolean waitUntilLeader,
 112                                                      final EffectiveModelContext schemaContext,
 113                                                      final String... shardNames) throws Exception {
 114         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 115         final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
 116
 117         setDataStoreName(typeName);
 118
 119         final DatastoreContext datastoreContext = datastoreContextBuilder.build();
 120         final DatastoreContextFactory mockContextFactory = mock(DatastoreContextFactory.class);
 121         doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 122         doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(anyString());
 123
 124         final Constructor<? extends AbstractDataStore> constructor = implementation.getDeclaredConstructor(
 125                 ActorSystem.class, ClusterWrapper.class, Configuration.class,
 126                 DatastoreContextFactory.class, DatastoreSnapshot.class);
 127
 128         final AbstractDataStore dataStore = constructor.newInstance(getSystem(), cluster, config, mockContextFactory,
 129             restoreFromSnapshot);
 130
 131         dataStore.onModelContextUpdated(schemaContext);
 132
 133         if (waitUntilLeader) {
 134             waitUntilLeader(dataStore.getActorUtils(), shardNames);
 135         }
 136
 137         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 138         return dataStore;
 139     }
 140
 141     private void setDataStoreName(final String typeName) {
 142         if ("config".equals(typeName)) {
 143             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.CONFIGURATION);
 144         } else if ("operational".equals(typeName)) {
 145             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.OPERATIONAL);
 146         } else {
 147             datastoreContextBuilder.dataStoreName(typeName);
 148         }
 149     }
 150
 151     public void waitUntilLeader(final ActorUtils actorUtils, final String... shardNames) {
 152         for (String shardName: shardNames) {
 153             ActorRef shard = findLocalShard(actorUtils, shardName);
 154
 155             assertNotNull("Shard was not created for " + shardName, shard);
 156
 157             waitUntilLeader(shard);
 158         }
 159     }
 160
 161     public void waitUntilNoLeader(final ActorUtils actorUtils, final String... shardNames) {
 162         for (String shardName: shardNames) {
 163             ActorRef shard = findLocalShard(actorUtils, shardName);
 164             assertNotNull("No local shard found for " + shardName, shard);
 165
 166             waitUntilNoLeader(shard);
 167         }
 168     }
 169
 170     public void waitForMembersUp(final String... otherMembers) {
 171         Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
 172         Stopwatch sw = Stopwatch.createStarted();
 173         while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
 174             CurrentClusterState state = Cluster.get(getSystem()).state();
 175             for (Member m: state.getMembers()) {
 176                 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
 177                         && otherMembersSet.isEmpty()) {
 178                     return;
 179                 }
 180             }
 181
 182             Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
 183         }
 184
 185         fail("Member(s) " + otherMembersSet + " are not Up");
 186     }
 187
 188     public static ActorRef findLocalShard(final ActorUtils actorUtils, final String shardName) {
 189         for (int i = 0; i < 20 * 5; i++) {
 190             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 191             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
 192             if (shardReply.isPresent()) {
 193                 return shardReply.orElseThrow();
 194             }
 195         }
 196         return null;
 197     }
 198
 199     public static void waitUntilShardIsDown(final ActorUtils actorUtils, final String shardName) {
 200         for (int i = 0; i < 20 * 5 ; i++) {
 201             LOG.debug("Waiting for shard down {}", shardName);
 202             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 203             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
 204             if (!shardReply.isPresent()) {
 205                 return;
 206             }
 207         }
 208
 209         throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
 210     }
 211
 212     public static void verifyShardStats(final AbstractDataStore datastore, final String shardName,
 213             final ShardStatsVerifier verifier) throws Exception {
 214         ActorUtils actorUtils = datastore.getActorUtils();
 215
 216         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
 217         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
 218
 219         AssertionError lastError = null;
 220         Stopwatch sw = Stopwatch.createStarted();
 221         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 222             ShardStats shardStats = (ShardStats)actorUtils
 223                     .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
 224
 225             try {
 226                 verifier.verify(shardStats);
 227                 return;
 228             } catch (AssertionError e) {
 229                 lastError = e;
 230                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 231             }
 232         }
 233
 234         throw lastError;
 235     }
 236
 237     public static void verifyShardState(final AbstractDataStore datastore, final String shardName,
 238             final Consumer<OnDemandShardState> verifier) throws Exception {
 239         ActorUtils actorUtils = datastore.getActorUtils();
 240
 241         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
 242         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
 243
 244         AssertionError lastError = null;
 245         Stopwatch sw = Stopwatch.createStarted();
 246         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 247             OnDemandShardState shardState = (OnDemandShardState)actorUtils
 248                     .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
 249
 250             try {
 251                 verifier.accept(shardState);
 252                 return;
 253             } catch (AssertionError e) {
 254                 lastError = e;
 255                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 256             }
 257         }
 258
 259         throw lastError;
 260     }
 261
 262     void testWriteTransaction(final AbstractDataStore dataStore, final YangInstanceIdentifier nodePath,
 263             final NormalizedNode nodeToWrite) throws Exception {
 264
 265         // 1. Create a write-only Tx
 266
 267         DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
 268         assertNotNull("newWriteOnlyTransaction returned null", writeTx);
 269
 270         // 2. Write some data
 271
 272         writeTx.write(nodePath, nodeToWrite);
 273
 274         // 3. Ready the Tx for commit
 275
 276         DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
 277
 278         // 4. Commit the Tx
 279
 280         doCommit(cohort);
 281
 282         // 5. Verify the data in the store
 283
 284         DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
 285         assertEquals(Optional.of(nodeToWrite), readTx.read(nodePath).get(5, TimeUnit.SECONDS));
 286     }
 287
 288     public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
 289         Boolean canCommit = cohort.canCommit().get(commitTimeout, TimeUnit.SECONDS);
 290         assertEquals("canCommit", Boolean.TRUE, canCommit);
 291         cohort.preCommit().get(5, TimeUnit.SECONDS);
 292         cohort.commit().get(5, TimeUnit.SECONDS);
 293     }
 294
 295     void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
 296             throws Exception {
 297         Boolean canCommit = canCommitFuture.get(commitTimeout, TimeUnit.SECONDS);
 298         assertEquals("canCommit", Boolean.TRUE, canCommit);
 299         cohort.preCommit().get(5, TimeUnit.SECONDS);
 300         cohort.commit().get(5, TimeUnit.SECONDS);
 301     }
 302
 303     void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
 304             final Class<? extends Exception> expType) {
 305         assertThrows(expType, () -> txChain.newWriteOnlyTransaction());
 306         assertThrows(expType, () -> txChain.newReadWriteTransaction());
 307         assertThrows(expType, () -> txChain.newReadOnlyTransaction());
 308     }
 309
 310     public interface ShardStatsVerifier {
 311         void verify(ShardStats stats);
 312     }
 313 }