2 * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore;
11 import akka.actor.ActorRef;
12 import akka.actor.ActorSelection;
13 import akka.actor.Props;
14 import akka.event.Logging;
15 import akka.event.LoggingAdapter;
16 import akka.japi.Creator;
17 import akka.serialization.Serialization;
18 import com.google.common.base.Optional;
19 import com.google.common.util.concurrent.ListenableFuture;
20 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardMBeanFactory;
21 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
22 import org.opendaylight.controller.cluster.datastore.messages.CommitTransactionReply;
23 import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction;
24 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionChain;
25 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionChainReply;
26 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply;
27 import org.opendaylight.controller.cluster.datastore.messages.EnableNotification;
28 import org.opendaylight.controller.cluster.datastore.messages.ForwardedCommitTransaction;
29 import org.opendaylight.controller.cluster.datastore.messages.PeerAddressResolved;
30 import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListener;
31 import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListenerReply;
32 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
33 import org.opendaylight.controller.cluster.datastore.modification.Modification;
34 import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification;
35 import org.opendaylight.controller.cluster.raft.ConfigParams;
36 import org.opendaylight.controller.cluster.raft.DefaultConfigParamsImpl;
37 import org.opendaylight.controller.cluster.raft.RaftActor;
38 import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry;
39 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeListener;
40 import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStore;
41 import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStoreFactory;
42 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadWriteTransaction;
43 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
44 import org.opendaylight.controller.sal.core.spi.data.DOMStoreTransactionChain;
45 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
46 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
47 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
48 import scala.concurrent.duration.FiniteDuration;
50 import java.util.ArrayList;
51 import java.util.Date;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.concurrent.ExecutionException;
56 import java.util.concurrent.TimeUnit;
59 * A Shard represents a portion of the logical data tree <br/>
61 * Our Shard uses InMemoryDataStore as it's internal representation and delegates all requests it
64 public class Shard extends RaftActor {
66 private static final ConfigParams configParams = new ShardConfigParams();
68 public static final String DEFAULT_NAME = "default";
70 private final InMemoryDOMDataStore store;
72 private final Map<Object, DOMStoreThreePhaseCommitCohort>
73 modificationToCohort = new HashMap<>();
75 private final LoggingAdapter LOG =
76 Logging.getLogger(getContext().system(), this);
78 // By default persistent will be true and can be turned off using the system
79 // property persistent
80 private final boolean persistent;
82 private final String name;
84 private volatile SchemaContext schemaContext;
86 private final ShardStats shardMBean;
88 private final List<ActorSelection> dataChangeListeners = new ArrayList<>();
90 private Shard(String name, Map<String, String> peerAddresses) {
91 super(name, peerAddresses, Optional.of(configParams));
95 String setting = System.getProperty("shard.persistent");
97 this.persistent = !"false".equals(setting);
99 LOG.info("Creating shard : {} persistent : {}", name, persistent);
101 store = InMemoryDOMDataStoreFactory.create(name, null);
103 shardMBean = ShardMBeanFactory.getShardStatsMBean(name);
107 public static Props props(final String name,
108 final Map<String, String> peerAddresses) {
109 return Props.create(new Creator<Shard>() {
112 public Shard create() throws Exception {
113 return new Shard(name, peerAddresses);
120 @Override public void onReceiveCommand(Object message) {
121 LOG.debug("Received message {} from {}", message.getClass().toString(),
124 if (message.getClass()
125 .equals(CreateTransactionChain.SERIALIZABLE_CLASS)) {
127 createTransactionChain();
128 } else if (getLeader() != null) {
129 getLeader().forward(message, getContext());
131 } else if (message instanceof RegisterChangeListener) {
132 registerChangeListener((RegisterChangeListener) message);
133 } else if (message instanceof UpdateSchemaContext) {
134 updateSchemaContext((UpdateSchemaContext) message);
135 } else if (message instanceof ForwardedCommitTransaction) {
136 handleForwardedCommit((ForwardedCommitTransaction) message);
137 } else if (message.getClass()
138 .equals(CreateTransaction.SERIALIZABLE_CLASS)) {
140 createTransaction(CreateTransaction.fromSerializable(message));
141 } else if (getLeader() != null) {
142 getLeader().forward(message, getContext());
144 } else if (message instanceof PeerAddressResolved) {
145 PeerAddressResolved resolved = (PeerAddressResolved) message;
146 setPeerAddress(resolved.getPeerId(), resolved.getPeerAddress());
148 super.onReceiveCommand(message);
152 private ActorRef createTypedTransactionActor(
153 CreateTransaction createTransaction, String transactionId) {
154 if (createTransaction.getTransactionType()
155 == TransactionProxy.TransactionType.READ_ONLY.ordinal()) {
156 shardMBean.incrementReadOnlyTransactionCount();
157 return getContext().actorOf(
159 .props(store.newReadOnlyTransaction(), getSelf(),
160 schemaContext), transactionId);
162 } else if (createTransaction.getTransactionType()
163 == TransactionProxy.TransactionType.READ_WRITE.ordinal()) {
164 shardMBean.incrementReadWriteTransactionCount();
165 return getContext().actorOf(
167 .props(store.newReadWriteTransaction(), getSelf(),
168 schemaContext), transactionId);
171 } else if (createTransaction.getTransactionType()
172 == TransactionProxy.TransactionType.WRITE_ONLY.ordinal()) {
173 shardMBean.incrementWriteOnlyTransactionCount();
174 return getContext().actorOf(
176 .props(store.newWriteOnlyTransaction(), getSelf(),
177 schemaContext), transactionId);
179 throw new IllegalArgumentException(
180 "CreateTransaction message has unidentified transaction type="
181 + createTransaction.getTransactionType());
185 private void createTransaction(CreateTransaction createTransaction) {
187 String transactionId = "shard-" + createTransaction.getTransactionId();
188 LOG.info("Creating transaction : {} ", transactionId);
189 ActorRef transactionActor =
190 createTypedTransactionActor(createTransaction, transactionId);
193 .tell(new CreateTransactionReply(
194 Serialization.serializedActorPath(transactionActor),
195 createTransaction.getTransactionId()).toSerializable(),
199 private void commit(final ActorRef sender, Object serialized) {
200 Modification modification = MutableCompositeModification
201 .fromSerializable(serialized, schemaContext);
202 DOMStoreThreePhaseCommitCohort cohort =
203 modificationToCohort.remove(serialized);
204 if (cohort == null) {
206 "Could not find cohort for modification : {}", modification);
207 LOG.info("Writing modification using a new transaction");
208 DOMStoreReadWriteTransaction transaction =
209 store.newReadWriteTransaction();
210 modification.apply(transaction);
211 DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready();
212 ListenableFuture<Void> future =
213 commitCohort.preCommit();
216 future = commitCohort.commit();
218 } catch (InterruptedException | ExecutionException e) {
219 shardMBean.incrementFailedTransactionsCount();
220 LOG.error("Failed to commit", e);
223 //we want to just apply the recovery commit and return
224 shardMBean.incrementCommittedTransactionCount();
228 final ListenableFuture<Void> future = cohort.commit();
229 final ActorRef self = getSelf();
230 future.addListener(new Runnable() {
236 .tell(new CommitTransactionReply().toSerializable(),
238 shardMBean.incrementCommittedTransactionCount();
239 shardMBean.setLastCommittedTransactionTime(new Date());
241 } catch (InterruptedException | ExecutionException e) {
242 shardMBean.incrementFailedTransactionsCount();
243 // FIXME : Handle this properly
244 LOG.error(e, "An exception happened when committing");
247 }, getContext().dispatcher());
250 private void handleForwardedCommit(ForwardedCommitTransaction message) {
251 Object serializedModification =
252 message.getModification().toSerializable();
255 .put(serializedModification, message.getCohort());
258 this.persistData(getSender(), "identifier",
259 new CompositeModificationPayload(serializedModification));
261 this.commit(getSender(), serializedModification);
265 private void updateSchemaContext(UpdateSchemaContext message) {
266 this.schemaContext = message.getSchemaContext();
267 store.onGlobalContextUpdated(message.getSchemaContext());
270 private void registerChangeListener(
271 RegisterChangeListener registerChangeListener) {
273 LOG.debug("registerDataChangeListener for " + registerChangeListener
277 ActorSelection dataChangeListenerPath = getContext()
278 .system().actorSelection(
279 registerChangeListener.getDataChangeListenerPath());
282 // Notify the listener if notifications should be enabled or not
283 // If this shard is the leader then it will enable notifications else
285 dataChangeListenerPath
286 .tell(new EnableNotification(isLeader()), getSelf());
288 // Now store a reference to the data change listener so it can be notified
289 // at a later point if notifications should be enabled or disabled
290 dataChangeListeners.add(dataChangeListenerPath);
292 AsyncDataChangeListener<YangInstanceIdentifier, NormalizedNode<?, ?>>
294 new DataChangeListenerProxy(schemaContext, dataChangeListenerPath);
296 org.opendaylight.yangtools.concepts.ListenerRegistration<AsyncDataChangeListener<YangInstanceIdentifier, NormalizedNode<?, ?>>>
298 store.registerChangeListener(registerChangeListener.getPath(),
299 listener, registerChangeListener.getScope());
300 ActorRef listenerRegistration =
301 getContext().actorOf(
302 DataChangeListenerRegistration.props(registration));
305 "registerDataChangeListener sending reply, listenerRegistrationPath = "
306 + listenerRegistration.path().toString());
309 .tell(new RegisterChangeListenerReply(listenerRegistration.path()),
313 private void createTransactionChain() {
314 DOMStoreTransactionChain chain = store.createTransactionChain();
315 ActorRef transactionChain =
316 getContext().actorOf(
317 ShardTransactionChain.props(chain, schemaContext));
319 .tell(new CreateTransactionChainReply(transactionChain.path())
324 @Override protected void applyState(ActorRef clientActor, String identifier,
327 if (data instanceof CompositeModificationPayload) {
328 Object modification =
329 ((CompositeModificationPayload) data).getModification();
331 if (modification != null) {
332 commit(clientActor, modification);
334 LOG.error("modification is null - this is very unexpected");
339 LOG.error("Unknown state received {}", data);
342 ReplicatedLogEntry lastLogEntry = getLastLogEntry();
344 if(lastLogEntry != null){
345 shardMBean.setLastLogIndex(lastLogEntry.getIndex());
346 shardMBean.setLastLogTerm(lastLogEntry.getTerm());
349 shardMBean.setCommitIndex(getCommitIndex());
350 shardMBean.setLastApplied(getLastApplied());
354 @Override protected Object createSnapshot() {
355 throw new UnsupportedOperationException("createSnapshot");
358 @Override protected void applySnapshot(Object snapshot) {
359 throw new UnsupportedOperationException("applySnapshot");
362 @Override protected void onStateChanged() {
363 for (ActorSelection dataChangeListener : dataChangeListeners) {
365 .tell(new EnableNotification(isLeader()), getSelf());
368 if (getLeaderId() != null) {
369 shardMBean.setLeader(getLeaderId());
372 shardMBean.setRaftState(getRaftState().name());
373 shardMBean.setCurrentTerm(getCurrentTerm());
376 @Override public String persistenceId() {
381 private static class ShardConfigParams extends DefaultConfigParamsImpl {
382 public static final FiniteDuration HEART_BEAT_INTERVAL =
383 new FiniteDuration(500, TimeUnit.MILLISECONDS);
385 @Override public FiniteDuration getHeartBeatInterval() {
386 return HEART_BEAT_INTERVAL;