X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FShard.java;h=7d570046d406feec976620f9215398475711a756;hb=4ef563c481b83e360e688a59ac346b8328870d58;hp=6a6a181b6c03ac744d02a3e8e815011d2cf99c3f;hpb=e71922c94cec22e9f37648a2d04bf2eb3274cf2f;p=controller.git diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java index 6a6a181b6c..7d570046d4 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java @@ -10,19 +10,22 @@ package org.opendaylight.controller.cluster.datastore; import akka.actor.ActorRef; import akka.actor.ActorSelection; +import akka.actor.PoisonPill; import akka.actor.Props; import akka.event.Logging; import akka.event.LoggingAdapter; import akka.japi.Creator; import akka.persistence.RecoveryFailure; import akka.serialization.Serialization; - +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.CheckedFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier; import org.opendaylight.controller.cluster.datastore.identifiers.ShardTransactionIdentifier; import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardMBeanFactory; @@ -35,30 +38,35 @@ import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionR import org.opendaylight.controller.cluster.datastore.messages.EnableNotification; import org.opendaylight.controller.cluster.datastore.messages.ForwardedCommitTransaction; import org.opendaylight.controller.cluster.datastore.messages.PeerAddressResolved; +import org.opendaylight.controller.cluster.datastore.messages.ReadData; +import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListener; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListenerReply; import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext; import org.opendaylight.controller.cluster.datastore.modification.Modification; import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification; +import org.opendaylight.controller.cluster.datastore.node.NormalizedNodeToNodeCodec; import org.opendaylight.controller.cluster.raft.ConfigParams; import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl; import org.opendaylight.controller.cluster.raft.RaftActor; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; +import org.opendaylight.controller.cluster.raft.base.messages.CaptureSnapshotReply; import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeListener; +import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStore; import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStoreFactory; -import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadWriteTransaction; +import org.opendaylight.controller.protobuff.messages.common.NormalizedNodeMessages; +import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadTransaction; import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort; import org.opendaylight.controller.sal.core.spi.data.DOMStoreTransactionChain; +import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction; import org.opendaylight.yangtools.concepts.ListenerRegistration; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; - import scala.concurrent.duration.FiniteDuration; import java.util.ArrayList; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -102,12 +110,15 @@ public class Shard extends RaftActor { private SchemaContext schemaContext; + private ActorRef createSnapshotTransaction; + private Shard(ShardIdentifier name, Map peerAddresses, - DatastoreContext datastoreContext) { + DatastoreContext datastoreContext, SchemaContext schemaContext) { super(name.toString(), mapPeerAddresses(peerAddresses), Optional.of(configParams)); this.name = name; this.datastoreContext = datastoreContext; + this.schemaContext = schemaContext; String setting = System.getProperty("shard.persistent"); @@ -118,8 +129,14 @@ public class Shard extends RaftActor { store = InMemoryDOMDataStoreFactory.create(name.toString(), null, datastoreContext.getDataStoreProperties()); - shardMBean = ShardMBeanFactory.getShardStatsMBean(name.toString()); + if(schemaContext != null) { + store.onGlobalContextUpdated(schemaContext); + } + shardMBean = ShardMBeanFactory.getShardStatsMBean(name.toString(), + datastoreContext.getDataStoreMXBeanType()); + shardMBean.setDataStoreExecutor(store.getDomStoreExecutor()); + shardMBean.setNotificationManager(store.getDataChangeListenerNotificationManager()); } @@ -137,12 +154,13 @@ public class Shard extends RaftActor { public static Props props(final ShardIdentifier name, final Map peerAddresses, - DatastoreContext datastoreContext) { + DatastoreContext datastoreContext, SchemaContext schemaContext) { Preconditions.checkNotNull(name, "name should not be null"); Preconditions.checkNotNull(peerAddresses, "peerAddresses should not be null"); - Preconditions.checkNotNull(datastoreContext, "shardContext should not be null"); + Preconditions.checkNotNull(datastoreContext, "dataStoreContext should not be null"); + Preconditions.checkNotNull(schemaContext, "schemaContext should not be null"); - return Props.create(new ShardCreator(name, peerAddresses, datastoreContext)); + return Props.create(new ShardCreator(name, peerAddresses, datastoreContext, schemaContext)); } @Override public void onReceiveRecover(Object message) { @@ -167,6 +185,15 @@ public class Shard extends RaftActor { } else if (getLeader() != null) { getLeader().forward(message, getContext()); } + } else if(message.getClass().equals(ReadDataReply.SERIALIZABLE_CLASS)) { + // This must be for install snapshot. Don't want to open this up and trigger + // deSerialization + self().tell(new CaptureSnapshotReply(ReadDataReply.getNormalizedNodeByteString(message)), self()); + + // Send a PoisonPill instead of sending close transaction because we do not really need + // a response + getSender().tell(PoisonPill.getInstance(), self()); + } else if (message instanceof RegisterChangeListener) { registerChangeListener((RegisterChangeListener) message); } else if (message instanceof UpdateSchemaContext) { @@ -190,60 +217,79 @@ public class Shard extends RaftActor { } private ActorRef createTypedTransactionActor( - CreateTransaction createTransaction, + int transactionType, ShardTransactionIdentifier transactionId) { - if (createTransaction.getTransactionType() + + if(this.schemaContext == null){ + throw new NullPointerException("schemaContext should not be null"); + } + + if (transactionType == TransactionProxy.TransactionType.READ_ONLY.ordinal()) { shardMBean.incrementReadOnlyTransactionCount(); return getContext().actorOf( ShardTransaction.props(store.newReadOnlyTransaction(), getSelf(), - schemaContext,datastoreContext, name.toString()), transactionId.toString()); + schemaContext,datastoreContext, shardMBean), transactionId.toString()); - } else if (createTransaction.getTransactionType() + } else if (transactionType == TransactionProxy.TransactionType.READ_WRITE.ordinal()) { shardMBean.incrementReadWriteTransactionCount(); return getContext().actorOf( ShardTransaction.props(store.newReadWriteTransaction(), getSelf(), - schemaContext, datastoreContext,name.toString()), transactionId.toString()); + schemaContext, datastoreContext, shardMBean), transactionId.toString()); - } else if (createTransaction.getTransactionType() + } else if (transactionType == TransactionProxy.TransactionType.WRITE_ONLY.ordinal()) { shardMBean.incrementWriteOnlyTransactionCount(); return getContext().actorOf( ShardTransaction.props(store.newWriteOnlyTransaction(), getSelf(), - schemaContext, datastoreContext, name.toString()), transactionId.toString()); + schemaContext, datastoreContext, shardMBean), transactionId.toString()); } else { throw new IllegalArgumentException( "Shard="+name + ":CreateTransaction message has unidentified transaction type=" - + createTransaction.getTransactionType()); + + transactionType); } } private void createTransaction(CreateTransaction createTransaction) { + createTransaction(createTransaction.getTransactionType(), + createTransaction.getTransactionId()); + } + + private ActorRef createTransaction(int transactionType, String remoteTransactionId) { ShardTransactionIdentifier transactionId = ShardTransactionIdentifier.builder() - .remoteTransactionId(createTransaction.getTransactionId()) + .remoteTransactionId(remoteTransactionId) .build(); LOG.debug("Creating transaction : {} ", transactionId); ActorRef transactionActor = - createTypedTransactionActor(createTransaction, transactionId); + createTypedTransactionActor(transactionType, transactionId); getSender() .tell(new CreateTransactionReply( Serialization.serializedActorPath(transactionActor), - createTransaction.getTransactionId()).toSerializable(), - getSelf() - ); + remoteTransactionId).toSerializable(), + getSelf()); + + return transactionActor; } + private void syncCommitTransaction(DOMStoreWriteTransaction transaction) + throws ExecutionException, InterruptedException { + DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); + commitCohort.preCommit().get(); + commitCohort.commit().get(); + } + + private void commit(final ActorRef sender, Object serialized) { Modification modification = MutableCompositeModification .fromSerializable(serialized, schemaContext); @@ -253,16 +299,11 @@ public class Shard extends RaftActor { LOG.debug( "Could not find cohort for modification : {}. Writing modification using a new transaction", modification); - DOMStoreReadWriteTransaction transaction = - store.newReadWriteTransaction(); + DOMStoreWriteTransaction transaction = + store.newWriteOnlyTransaction(); modification.apply(transaction); - DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); - ListenableFuture future = - commitCohort.preCommit(); try { - future.get(); - future = commitCohort.commit(); - future.get(); + syncCommitTransaction(transaction); } catch (InterruptedException | ExecutionException e) { shardMBean.incrementFailedTransactionsCount(); LOG.error("Failed to commit", e); @@ -281,7 +322,7 @@ public class Shard extends RaftActor { public void onSuccess(Void v) { sender.tell(new CommitTransactionReply().toSerializable(), self); shardMBean.incrementCommittedTransactionCount(); - shardMBean.setLastCommittedTransactionTime(new Date()); + shardMBean.setLastCommittedTransactionTime(System.currentTimeMillis()); } @Override @@ -311,9 +352,14 @@ public class Shard extends RaftActor { private void updateSchemaContext(UpdateSchemaContext message) { this.schemaContext = message.getSchemaContext(); + updateSchemaContext(message.getSchemaContext()); store.onGlobalContextUpdated(message.getSchemaContext()); } + @VisibleForTesting void updateSchemaContext(SchemaContext schemaContext) { + store.onGlobalContextUpdated(schemaContext); + } + private void registerChangeListener( RegisterChangeListener registerChangeListener) { @@ -358,9 +404,9 @@ public class Shard extends RaftActor { private void createTransactionChain() { DOMStoreTransactionChain chain = store.createTransactionChain(); ActorRef transactionChain = getContext().actorOf( - ShardTransactionChain.props(chain, schemaContext, datastoreContext,name.toString() )); + ShardTransactionChain.props(chain, schemaContext, datastoreContext, shardMBean)); getSender().tell(new CreateTransactionChainReply(transactionChain.path()).toSerializable(), - getSelf()); + getSelf()); } @Override protected void applyState(ActorRef clientActor, String identifier, @@ -396,11 +442,39 @@ public class Shard extends RaftActor { } @Override protected void createSnapshot() { - throw new UnsupportedOperationException("createSnapshot"); + if (createSnapshotTransaction == null) { + + // Create a transaction. We are really going to treat the transaction as a worker + // so that this actor does not get block building the snapshot + createSnapshotTransaction = createTransaction( + TransactionProxy.TransactionType.READ_ONLY.ordinal(), + "createSnapshot"); + + createSnapshotTransaction.tell( + new ReadData(YangInstanceIdentifier.builder().build()).toSerializable(), self()); + + } } - @Override protected void applySnapshot(ByteString snapshot) { - throw new UnsupportedOperationException("applySnapshot"); + @VisibleForTesting @Override protected void applySnapshot(ByteString snapshot) { + // Since this will be done only on Recovery or when this actor is a Follower + // we can safely commit everything in here. We not need to worry about event notifications + // as they would have already been disabled on the follower + try { + DOMStoreWriteTransaction transaction = store.newWriteOnlyTransaction(); + NormalizedNodeMessages.Node serializedNode = NormalizedNodeMessages.Node.parseFrom(snapshot); + NormalizedNode node = new NormalizedNodeToNodeCodec(schemaContext) + .decode(YangInstanceIdentifier.builder().build(), serializedNode); + + // delete everything first + transaction.delete(YangInstanceIdentifier.builder().build()); + + // Add everything from the remote node back + transaction.write(YangInstanceIdentifier.builder().build(), node); + syncCommitTransaction(transaction); + } catch (InvalidProtocolBufferException | InterruptedException | ExecutionException e) { + LOG.error(e, "An exception occurred when applying snapshot"); + } } @Override protected void onStateChanged() { @@ -438,17 +512,42 @@ public class Shard extends RaftActor { final ShardIdentifier name; final Map peerAddresses; final DatastoreContext datastoreContext; + final SchemaContext schemaContext; ShardCreator(ShardIdentifier name, Map peerAddresses, - DatastoreContext datastoreContext) { + DatastoreContext datastoreContext, SchemaContext schemaContext) { this.name = name; this.peerAddresses = peerAddresses; this.datastoreContext = datastoreContext; + this.schemaContext = schemaContext; } @Override public Shard create() throws Exception { - return new Shard(name, peerAddresses, datastoreContext); + return new Shard(name, peerAddresses, datastoreContext, schemaContext); } } + + @VisibleForTesting NormalizedNode readStore() throws ExecutionException, InterruptedException { + DOMStoreReadTransaction transaction = store.newReadOnlyTransaction(); + + CheckedFuture>, ReadFailedException> future = + transaction.read(YangInstanceIdentifier.builder().build()); + + NormalizedNode node = future.get().get(); + + transaction.close(); + + return node; + } + + @VisibleForTesting void writeToStore(YangInstanceIdentifier id, NormalizedNode node) + throws ExecutionException, InterruptedException { + DOMStoreWriteTransaction transaction = store.newWriteOnlyTransaction(); + + transaction.write(id, node); + + syncCommitTransaction(transaction); + } + }