opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.junit.Assert.fail;
  13 import akka.actor.ActorRef;
  14 import akka.actor.ActorSystem;
  15 import akka.cluster.Cluster;
  16 import akka.cluster.ClusterEvent.CurrentClusterState;
  17 import akka.cluster.Member;
  18 import akka.cluster.MemberStatus;
  19 import com.google.common.base.Optional;
  20 import com.google.common.base.Stopwatch;
  21 import com.google.common.collect.Sets;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.Uninterruptibles;
  24 import java.util.Set;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.TimeUnit;
  27 import org.mockito.Mockito;
  28 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  29 import org.opendaylight.controller.cluster.datastore.config.Configuration;
  30 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  31 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
  32 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
  33 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
  34 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  35 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadTransaction;
  36 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
  37 import org.opendaylight.controller.sal.core.spi.data.DOMStoreTransactionChain;
  38 import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction;
  39 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  40 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  41 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
  42 import scala.concurrent.Await;
  43 import scala.concurrent.Future;
  44 import scala.concurrent.duration.Duration;
  45
  46 public class IntegrationTestKit extends ShardTestKit {
  47
  48     protected DatastoreContext.Builder datastoreContextBuilder;
  49     protected DatastoreSnapshot restoreFromSnapshot;
  50
  51     public IntegrationTestKit(ActorSystem actorSystem, Builder datastoreContextBuilder) {
  52         super(actorSystem);
  53         this.datastoreContextBuilder = datastoreContextBuilder;
  54     }
  55
  56     public DatastoreContext.Builder getDatastoreContextBuilder() {
  57         return datastoreContextBuilder;
  58     }
  59
  60     public DistributedDataStore setupDistributedDataStore(String typeName, String... shardNames) {
  61         return setupDistributedDataStore(typeName, "module-shards.conf", true, SchemaContextHelper.full(), shardNames);
  62     }
  63
  64     public DistributedDataStore setupDistributedDataStore(String typeName, boolean waitUntilLeader,
  65             String... shardNames) {
  66         return setupDistributedDataStore(typeName, "module-shards.conf", waitUntilLeader,
  67                 SchemaContextHelper.full(), shardNames);
  68     }
  69
  70     public DistributedDataStore setupDistributedDataStore(String typeName, String moduleShardsConfig,
  71             boolean waitUntilLeader, String... shardNames) {
  72         return setupDistributedDataStore(typeName, moduleShardsConfig, waitUntilLeader,
  73                 SchemaContextHelper.full(), shardNames);
  74     }
  75
  76     public DistributedDataStore setupDistributedDataStore(String typeName, String moduleShardsConfig,
  77             boolean waitUntilLeader, SchemaContext schemaContext, String... shardNames) {
  78         ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
  79         Configuration config = new ConfigurationImpl(moduleShardsConfig, "modules.conf");
  80
  81         datastoreContextBuilder.dataStoreName(typeName);
  82
  83         DatastoreContext datastoreContext = datastoreContextBuilder.build();
  84         DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
  85         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
  86         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
  87
  88         DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster, config, mockContextFactory,
  89                 restoreFromSnapshot);
  90
  91         dataStore.onGlobalContextUpdated(schemaContext);
  92
  93         if(waitUntilLeader) {
  94             waitUntilLeader(dataStore.getActorContext(), shardNames);
  95         }
  96
  97         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
  98         return dataStore;
  99     }
 100
 101     public void waitUntilLeader(ActorContext actorContext, String... shardNames) {
 102         for(String shardName: shardNames) {
 103             ActorRef shard = findLocalShard(actorContext, shardName);
 104
 105             assertNotNull("Shard was not created for " + shardName, shard);
 106
 107             waitUntilLeader(shard);
 108         }
 109     }
 110
 111     public void waitUntilNoLeader(ActorContext actorContext, String... shardNames) {
 112         for(String shardName: shardNames) {
 113             ActorRef shard = findLocalShard(actorContext, shardName);
 114             assertNotNull("No local shard found for " + shardName, shard);
 115
 116             waitUntilNoLeader(shard);
 117         }
 118     }
 119
 120     public void waitForMembersUp(String... otherMembers) {
 121         Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
 122         Stopwatch sw = Stopwatch.createStarted();
 123         while(sw.elapsed(TimeUnit.SECONDS) <= 10) {
 124             CurrentClusterState state = Cluster.get(getSystem()).state();
 125             for(Member m: state.getMembers()) {
 126                 if(m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next()) &&
 127                         otherMembersSet.isEmpty()) {
 128                     return;
 129                 }
 130             }
 131
 132             Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
 133         }
 134
 135         fail("Member(s) " + otherMembersSet + " are not Up");
 136     }
 137
 138     public static ActorRef findLocalShard(ActorContext actorContext, String shardName) {
 139         ActorRef shard = null;
 140         for(int i = 0; i < 20 * 5 && shard == null; i++) {
 141             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 142             Optional<ActorRef> shardReply = actorContext.findLocalShard(shardName);
 143             if(shardReply.isPresent()) {
 144                 shard = shardReply.get();
 145             }
 146         }
 147         return shard;
 148     }
 149
 150     public static void verifyShardStats(DistributedDataStore datastore, String shardName, ShardStatsVerifier verifier)
 151             throws Exception {
 152         ActorContext actorContext = datastore.getActorContext();
 153
 154         Future<ActorRef> future = actorContext.findLocalShardAsync(shardName);
 155         ActorRef shardActor = Await.result(future, Duration.create(10, TimeUnit.SECONDS));
 156
 157         AssertionError lastError = null;
 158         Stopwatch sw = Stopwatch.createStarted();
 159         while(sw.elapsed(TimeUnit.SECONDS) <= 5) {
 160             ShardStats shardStats = (ShardStats)actorContext.
 161                     executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
 162
 163             try {
 164                 verifier.verify(shardStats);
 165                 return;
 166             } catch (AssertionError e) {
 167                 lastError = e;
 168                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 169             }
 170         }
 171
 172         throw lastError;
 173     }
 174
 175     void testWriteTransaction(DistributedDataStore dataStore, YangInstanceIdentifier nodePath,
 176             NormalizedNode<?, ?> nodeToWrite) throws Exception {
 177
 178         // 1. Create a write-only Tx
 179
 180         DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
 181         assertNotNull("newWriteOnlyTransaction returned null", writeTx);
 182
 183         // 2. Write some data
 184
 185         writeTx.write(nodePath, nodeToWrite);
 186
 187         // 3. Ready the Tx for commit
 188
 189         DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
 190
 191         // 4. Commit the Tx
 192
 193         doCommit(cohort);
 194
 195         // 5. Verify the data in the store
 196
 197         DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
 198
 199         Optional<NormalizedNode<?, ?>> optional = readTx.read(nodePath).get(5, TimeUnit.SECONDS);
 200         assertEquals("isPresent", true, optional.isPresent());
 201         assertEquals("Data node", nodeToWrite, optional.get());
 202     }
 203
 204     public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
 205         Boolean canCommit = cohort.canCommit().get(7, TimeUnit.SECONDS);
 206         assertEquals("canCommit", true, canCommit);
 207         cohort.preCommit().get(5, TimeUnit.SECONDS);
 208         cohort.commit().get(5, TimeUnit.SECONDS);
 209     }
 210
 211     void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
 212         Boolean canCommit = canCommitFuture.get(7, TimeUnit.SECONDS);
 213         assertEquals("canCommit", true, canCommit);
 214         cohort.preCommit().get(5, TimeUnit.SECONDS);
 215         cohort.commit().get(5, TimeUnit.SECONDS);
 216     }
 217
 218     void assertExceptionOnCall(Callable<Void> callable, Class<? extends Exception> expType)
 219             throws Exception {
 220         try {
 221             callable.call();
 222             fail("Expected " + expType.getSimpleName());
 223         } catch(Exception e) {
 224             assertEquals("Exception type", expType, e.getClass());
 225         }
 226     }
 227
 228     void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
 229             Class<? extends Exception> expType) throws Exception {
 230         assertExceptionOnCall(new Callable<Void>() {
 231             @Override
 232             public Void call() throws Exception {
 233                 txChain.newWriteOnlyTransaction();
 234                 return null;
 235             }
 236         }, expType);
 237
 238         assertExceptionOnCall(new Callable<Void>() {
 239             @Override
 240             public Void call() throws Exception {
 241                 txChain.newReadWriteTransaction();
 242                 return null;
 243             }
 244         }, expType);
 245
 246         assertExceptionOnCall(new Callable<Void>() {
 247             @Override
 248             public Void call() throws Exception {
 249                 txChain.newReadOnlyTransaction();
 250                 return null;
 251             }
 252         }, expType);
 253     }
 254
 255     public interface ShardStatsVerifier {
 256         void verify(ShardStats stats);
 257     }
 258 }