opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore.entityownership;
   9
  10 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID;
  11 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NODE_ID;
  12 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_ID_NODE_ID;
  13 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_ID_QNAME;
  14 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_NODE_ID;
  15 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
  16 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNER_NODE_ID;
  17 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNER_QNAME;
  18 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_TYPES_PATH;
  19 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_TYPE_NODE_ID;
  20 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_TYPE_QNAME;
  21 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.candidateMapEntry;
  22 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.candidateNodeKey;
  23 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.candidatePath;
  24 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.createEntity;
  25 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityOwnersWithCandidate;
  26 import akka.actor.ActorRef;
  27 import akka.actor.ActorSelection;
  28 import akka.actor.Props;
  29 import akka.pattern.Patterns;
  30 import com.google.common.base.Optional;
  31 import com.google.common.base.Strings;
  32 import java.util.ArrayList;
  33 import java.util.Collection;
  34 import java.util.HashMap;
  35 import java.util.HashSet;
  36 import java.util.Map;
  37 import java.util.Set;
  38 import java.util.concurrent.TimeUnit;
  39 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  40 import org.opendaylight.controller.cluster.datastore.Shard;
  41 import org.opendaylight.controller.cluster.datastore.entityownership.messages.CandidateAdded;
  42 import org.opendaylight.controller.cluster.datastore.entityownership.messages.CandidateRemoved;
  43 import org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterCandidateLocal;
  44 import org.opendaylight.controller.cluster.datastore.entityownership.messages.RegisterListenerLocal;
  45 import org.opendaylight.controller.cluster.datastore.entityownership.messages.UnregisterCandidateLocal;
  46 import org.opendaylight.controller.cluster.datastore.entityownership.messages.UnregisterListenerLocal;
  47 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  48 import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications;
  49 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  50 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  51 import org.opendaylight.controller.cluster.datastore.messages.SuccessReply;
  52 import org.opendaylight.controller.cluster.datastore.modification.DeleteModification;
  53 import org.opendaylight.controller.cluster.datastore.modification.MergeModification;
  54 import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
  55 import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
  56 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  57 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument;
  58 import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild;
  59 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  60 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  61 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  62 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeSnapshot;
  63 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
  64 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
  65 import scala.concurrent.Future;
  66
  67 /**
  68  * Special Shard for EntityOwnership.
  69  *
  70  * @author Thomas Pantelis
  71  */
  72 class EntityOwnershipShard extends Shard {
  73     private final String localMemberName;
  74     private final EntityOwnershipShardCommitCoordinator commitCoordinator;
  75     private final EntityOwnershipListenerSupport listenerSupport;
  76     private final Set<String> downPeerMemberNames = new HashSet<>();
  77     private final Map<String, String> peerIdToMemberNames = new HashMap<>();
  78
  79     private static DatastoreContext noPersistenceDatastoreContext(DatastoreContext datastoreContext) {
  80         return DatastoreContext.newBuilderFrom(datastoreContext).persistent(false).build();
  81     }
  82
  83     protected EntityOwnershipShard(ShardIdentifier name, Map<String, String> peerAddresses,
  84             DatastoreContext datastoreContext, SchemaContext schemaContext, String localMemberName) {
  85         super(name, peerAddresses, noPersistenceDatastoreContext(datastoreContext), schemaContext);
  86         this.localMemberName = localMemberName;
  87         this.commitCoordinator = new EntityOwnershipShardCommitCoordinator(localMemberName, LOG);
  88         this.listenerSupport = new EntityOwnershipListenerSupport(getContext(), persistenceId());
  89
  90         for(String peerId: peerAddresses.keySet()) {
  91             ShardIdentifier shardId = ShardIdentifier.builder().fromShardIdString(peerId).build();
  92             peerIdToMemberNames.put(peerId, shardId.getMemberName());
  93         }
  94     }
  95
  96     @Override
  97     protected void onDatastoreContext(DatastoreContext context) {
  98         super.onDatastoreContext(noPersistenceDatastoreContext(context));
  99     }
 100
 101     @Override
 102     protected void onRecoveryComplete() {
 103         super.onRecoveryComplete();
 104
 105         new CandidateListChangeListener(getSelf(), persistenceId()).init(getDataStore());
 106         new EntityOwnerChangeListener(localMemberName, listenerSupport).init(getDataStore());
 107     }
 108
 109     @Override
 110     public void onReceiveCommand(final Object message) throws Exception {
 111         if(message instanceof RegisterCandidateLocal) {
 112             onRegisterCandidateLocal((RegisterCandidateLocal)message);
 113         } else if(message instanceof UnregisterCandidateLocal) {
 114             onUnregisterCandidateLocal((UnregisterCandidateLocal)message);
 115         } else if(message instanceof CandidateAdded){
 116             onCandidateAdded((CandidateAdded) message);
 117         } else if(message instanceof CandidateRemoved){
 118             onCandidateRemoved((CandidateRemoved) message);
 119         } else if(message instanceof PeerDown) {
 120             onPeerDown((PeerDown) message);
 121         } else if(message instanceof PeerUp) {
 122             onPeerUp((PeerUp) message);
 123         } if(message instanceof RegisterListenerLocal) {
 124             onRegisterListenerLocal((RegisterListenerLocal)message);
 125         } if(message instanceof UnregisterListenerLocal) {
 126             onUnregisterListenerLocal((UnregisterListenerLocal)message);
 127         } else if(!commitCoordinator.handleMessage(message, this)) {
 128             super.onReceiveCommand(message);
 129         }
 130     }
 131
 132     private void onRegisterCandidateLocal(RegisterCandidateLocal registerCandidate) {
 133         LOG.debug("{}: onRegisterCandidateLocal: {}", persistenceId(), registerCandidate);
 134
 135         listenerSupport.addEntityOwnershipListener(registerCandidate.getEntity(), registerCandidate.getCandidate());
 136
 137         NormalizedNode<?, ?> entityOwners = entityOwnersWithCandidate(registerCandidate.getEntity().getType(),
 138                 registerCandidate.getEntity().getId(), localMemberName);
 139         commitCoordinator.commitModification(new MergeModification(ENTITY_OWNERS_PATH, entityOwners), this);
 140
 141         getSender().tell(SuccessReply.INSTANCE, getSelf());
 142     }
 143
 144     private void onUnregisterCandidateLocal(UnregisterCandidateLocal unregisterCandidate) {
 145         LOG.debug("{}: onUnregisterCandidateLocal: {}", persistenceId(), unregisterCandidate);
 146
 147         Entity entity = unregisterCandidate.getEntity();
 148         listenerSupport.removeEntityOwnershipListener(entity, unregisterCandidate.getCandidate());
 149
 150         YangInstanceIdentifier candidatePath = candidatePath(entity.getType(), entity.getId(), localMemberName);
 151         commitCoordinator.commitModification(new DeleteModification(candidatePath), this);
 152
 153         getSender().tell(SuccessReply.INSTANCE, getSelf());
 154     }
 155
 156     private void onRegisterListenerLocal(final RegisterListenerLocal registerListener) {
 157         LOG.debug("{}: onRegisterListenerLocal: {}", persistenceId(), registerListener);
 158
 159         listenerSupport.addEntityOwnershipListener(registerListener.getEntityType(), registerListener.getListener());
 160
 161         getSender().tell(SuccessReply.INSTANCE, getSelf());
 162
 163         searchForEntitiesOwnedBy(localMemberName, new EntityWalker() {
 164             @Override
 165             public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
 166                 Optional<DataContainerChild<? extends PathArgument, ?>> possibleType =
 167                         entityTypeNode.getChild(ENTITY_TYPE_NODE_ID);
 168                 String entityType = possibleType.isPresent() ? possibleType.get().getValue().toString() : null;
 169                 if(registerListener.getEntityType().equals(entityType)) {
 170                     Entity entity = new Entity(entityType,
 171                             (YangInstanceIdentifier) entityNode.getChild(ENTITY_ID_NODE_ID).get().getValue());
 172                     listenerSupport.notifyEntityOwnershipListener(entity, false, true, true, registerListener.getListener());
 173                 }
 174             }
 175         });
 176     }
 177
 178     private void onUnregisterListenerLocal(UnregisterListenerLocal unregisterListener) {
 179         LOG.debug("{}: onUnregisterListenerLocal: {}", persistenceId(), unregisterListener);
 180
 181         listenerSupport.removeEntityOwnershipListener(unregisterListener.getEntityType(), unregisterListener.getListener());
 182
 183         getSender().tell(SuccessReply.INSTANCE, getSelf());
 184     }
 185
 186     void tryCommitModifications(final BatchedModifications modifications) {
 187         if(isLeader()) {
 188             LOG.debug("{}: Committing BatchedModifications {} locally", persistenceId(), modifications.getTransactionID());
 189
 190             // Note that it's possible the commit won't get consensus and will timeout and not be applied
 191             // to the state. However we don't need to retry it in that case b/c it will be committed to
 192             // the journal first and, once a majority of followers come back on line and it is replicated,
 193             // it will be applied at that point.
 194             handleBatchedModificationsLocal(modifications, self());
 195         } else {
 196             final ActorSelection leader = getLeader();
 197             if (leader != null) {
 198                 if(LOG.isDebugEnabled()) {
 199                     LOG.debug("{}: Sending BatchedModifications {} to leader {}", persistenceId(),
 200                             modifications.getTransactionID(), leader);
 201                 }
 202
 203                 Future<Object> future = Patterns.ask(leader, modifications, TimeUnit.SECONDS.toMillis(
 204                         getDatastoreContext().getShardTransactionCommitTimeoutInSeconds()));
 205
 206                 Patterns.pipe(future, getContext().dispatcher()).pipeTo(getSelf(), ActorRef.noSender());
 207             }
 208         }
 209     }
 210
 211     boolean hasLeader() {
 212         return getLeader() != null && !isIsolatedLeader();
 213     }
 214
 215     @Override
 216     protected void onStateChanged() {
 217         super.onStateChanged();
 218
 219         commitCoordinator.onStateChanged(this, isLeader());
 220     }
 221
 222     @Override
 223     protected void onLeaderChanged(String oldLeader, String newLeader) {
 224         super.onLeaderChanged(oldLeader, newLeader);
 225
 226         LOG.debug("{}: onLeaderChanged: oldLeader: {}, newLeader: {}, isLeader: {}", persistenceId(), oldLeader,
 227                 newLeader, isLeader());
 228
 229         if(isLeader()) {
 230             // We were just elected leader. If the old leader is down, select new owners for the entities
 231             // owned by the down leader.
 232
 233             String oldLeaderMemberName = peerIdToMemberNames.get(oldLeader);
 234
 235             LOG.debug("{}: oldLeaderMemberName: {}", persistenceId(), oldLeaderMemberName);
 236
 237             if(downPeerMemberNames.contains(oldLeaderMemberName)) {
 238                 selectNewOwnerForEntitiesOwnedBy(oldLeaderMemberName);
 239             }
 240         }
 241     }
 242
 243     private void onCandidateRemoved(CandidateRemoved message) {
 244         LOG.debug("{}: onCandidateRemoved: {}", persistenceId(), message);
 245
 246         if(isLeader()) {
 247             String currentOwner = getCurrentOwner(message.getEntityPath());
 248             if(message.getRemovedCandidate().equals(currentOwner)){
 249                 writeNewOwner(message.getEntityPath(), newOwner(message.getRemainingCandidates()));
 250             }
 251         } else {
 252             // We're not the leader. If the removed candidate is our local member then check if we actually
 253             // have a local candidate registered. If we do then we must have been partitioned from the leader
 254             // and the leader removed our candidate since the leader can't tell the difference between a
 255             // temporary network partition and a node's process actually restarted. So, in that case, re-add
 256             // our candidate.
 257             if(localMemberName.equals(message.getRemovedCandidate()) &&
 258                     listenerSupport.hasCandidateForEntity(createEntity(message.getEntityPath()))) {
 259                 LOG.debug("Local candidate member was removed but a local candidate is registered for {}" +
 260                     " - adding back local candidate", message.getEntityPath());
 261
 262                 commitCoordinator.commitModification(new MergeModification(
 263                         candidatePath(message.getEntityPath(), localMemberName),
 264                         candidateMapEntry(localMemberName)), this);
 265             }
 266         }
 267     }
 268
 269     private void onCandidateAdded(CandidateAdded message) {
 270         if(!isLeader()){
 271             return;
 272         }
 273
 274         LOG.debug("{}: onCandidateAdded: {}", persistenceId(), message);
 275
 276         // Since a node's candidate member is only added by the node itself, we can assume the node is up so
 277         // remove it from the downPeerMemberNames.
 278         downPeerMemberNames.remove(message.getNewCandidate());
 279
 280         String currentOwner = getCurrentOwner(message.getEntityPath());
 281         if(Strings.isNullOrEmpty(currentOwner)){
 282             writeNewOwner(message.getEntityPath(), newOwner(message.getAllCandidates()));
 283         }
 284     }
 285
 286     private void onPeerDown(PeerDown peerDown) {
 287         LOG.info("{}: onPeerDown: {}", persistenceId(), peerDown);
 288
 289         String downMemberName = peerDown.getMemberName();
 290         if(downPeerMemberNames.add(downMemberName) && isLeader()) {
 291             // Remove the down peer as a candidate from all entities.
 292             removeCandidateFromEntities(downMemberName);
 293         }
 294     }
 295
 296     private void onPeerUp(PeerUp peerUp) {
 297         LOG.debug("{}: onPeerUp: {}", persistenceId(), peerUp);
 298
 299         peerIdToMemberNames.put(peerUp.getPeerId(), peerUp.getMemberName());
 300         downPeerMemberNames.remove(peerUp.getMemberName());
 301     }
 302
 303     private void selectNewOwnerForEntitiesOwnedBy(String owner) {
 304         final BatchedModifications modifications = commitCoordinator.newBatchedModifications();
 305         searchForEntitiesOwnedBy(owner, new EntityWalker() {
 306             @Override
 307             public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
 308                 Object newOwner = newOwner(getCandidateNames(entityNode));
 309                 YangInstanceIdentifier entityPath = YangInstanceIdentifier.builder(ENTITY_TYPES_PATH).
 310                         node(entityTypeNode.getIdentifier()).node(ENTITY_NODE_ID).node(entityNode.getIdentifier()).
 311                         node(ENTITY_OWNER_NODE_ID).build();
 312
 313                 LOG.debug("{}: Found entity {}, writing new owner {}", persistenceId(), entityPath, newOwner);
 314
 315                 modifications.addModification(new WriteModification(entityPath,
 316                         ImmutableNodes.leafNode(ENTITY_OWNER_NODE_ID, newOwner)));
 317             }
 318         });
 319
 320         commitCoordinator.commitModifications(modifications, this);
 321     }
 322
 323     private void removeCandidateFromEntities(final String owner) {
 324         final BatchedModifications modifications = commitCoordinator.newBatchedModifications();
 325         searchForEntities(new EntityWalker() {
 326             @Override
 327             public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
 328                 if(hasCandidate(entityNode, owner)) {
 329                     YangInstanceIdentifier entityId =
 330                             (YangInstanceIdentifier)entityNode.getIdentifier().getKeyValues().get(ENTITY_ID_QNAME);
 331                     YangInstanceIdentifier candidatePath = candidatePath(
 332                             entityTypeNode.getIdentifier().getKeyValues().get(ENTITY_TYPE_QNAME).toString(),
 333                             entityId, owner);
 334
 335                     LOG.info("{}: Found entity {}, removing candidate {}, path {}", persistenceId(), entityId,
 336                             owner, candidatePath);
 337
 338                     modifications.addModification(new DeleteModification(candidatePath));
 339                 }
 340             }
 341         });
 342
 343         commitCoordinator.commitModifications(modifications, this);
 344     }
 345
 346     private boolean hasCandidate(MapEntryNode entity, String candidateName) {
 347         return ((MapNode)entity.getChild(CANDIDATE_NODE_ID).get()).getChild(candidateNodeKey(candidateName)).isPresent();
 348     }
 349
 350     private void searchForEntitiesOwnedBy(final String owner, final EntityWalker walker) {
 351         DataTreeSnapshot snapshot = getDataStore().getDataTree().takeSnapshot();
 352         Optional<NormalizedNode<?, ?>> possibleEntityTypes = snapshot.readNode(ENTITY_TYPES_PATH);
 353         if(!possibleEntityTypes.isPresent()) {
 354             return;
 355         }
 356
 357         LOG.debug("{}: Searching for entities owned by {}", persistenceId(), owner);
 358
 359         searchForEntities(new EntityWalker() {
 360             @Override
 361             public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
 362                 Optional<DataContainerChild<? extends PathArgument, ?>> possibleOwner =
 363                         entityNode.getChild(ENTITY_OWNER_NODE_ID);
 364                 if(possibleOwner.isPresent() && owner.equals(possibleOwner.get().getValue().toString())) {
 365                     walker.onEntity(entityTypeNode, entityNode);
 366                 }
 367             }
 368         });
 369     }
 370
 371     private void searchForEntities(EntityWalker walker) {
 372         DataTreeSnapshot snapshot = getDataStore().getDataTree().takeSnapshot();
 373         Optional<NormalizedNode<?, ?>> possibleEntityTypes = snapshot.readNode(ENTITY_TYPES_PATH);
 374         if(!possibleEntityTypes.isPresent()) {
 375             return;
 376         }
 377
 378         for(MapEntryNode entityType:  ((MapNode) possibleEntityTypes.get()).getValue()) {
 379             Optional<DataContainerChild<? extends PathArgument, ?>> possibleEntities =
 380                     entityType.getChild(ENTITY_NODE_ID);
 381             if(!possibleEntities.isPresent()) {
 382                 continue; // shouldn't happen but handle anyway
 383             }
 384
 385             for(MapEntryNode entity:  ((MapNode) possibleEntities.get()).getValue()) {
 386                 walker.onEntity(entityType, entity);
 387             }
 388         }
 389     }
 390
 391     private Collection<String> getCandidateNames(MapEntryNode entity) {
 392         Collection<MapEntryNode> candidates = ((MapNode)entity.getChild(CANDIDATE_NODE_ID).get()).getValue();
 393         Collection<String> candidateNames = new ArrayList<>(candidates.size());
 394         for(MapEntryNode candidate: candidates) {
 395             candidateNames.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString());
 396         }
 397
 398         return candidateNames;
 399     }
 400
 401     private void writeNewOwner(YangInstanceIdentifier entityPath, String newOwner) {
 402         LOG.debug("{}: Writing new owner {} for entity {}", persistenceId(), newOwner, entityPath);
 403
 404         commitCoordinator.commitModification(new WriteModification(entityPath.node(ENTITY_OWNER_QNAME),
 405                 ImmutableNodes.leafNode(ENTITY_OWNER_NODE_ID, newOwner)), this);
 406     }
 407
 408     private String newOwner(Collection<String> candidates) {
 409         for(String candidate: candidates) {
 410             if(!downPeerMemberNames.contains(candidate)) {
 411                 return candidate;
 412             }
 413         }
 414
 415         return "";
 416     }
 417
 418     private String getCurrentOwner(YangInstanceIdentifier entityId) {
 419         DataTreeSnapshot snapshot = getDataStore().getDataTree().takeSnapshot();
 420         Optional<NormalizedNode<?, ?>> optionalEntityOwner = snapshot.readNode(entityId.node(ENTITY_OWNER_QNAME));
 421         if(optionalEntityOwner.isPresent()){
 422             return optionalEntityOwner.get().getValue().toString();
 423         }
 424         return null;
 425     }
 426
 427     public static Props props(final ShardIdentifier name, final Map<String, String> peerAddresses,
 428             final DatastoreContext datastoreContext, final SchemaContext schemaContext, final String localMemberName) {
 429         return Props.create(new Creator(name, peerAddresses, datastoreContext, schemaContext, localMemberName));
 430     }
 431
 432     private static class Creator extends AbstractShardCreator {
 433         private static final long serialVersionUID = 1L;
 434
 435         private final String localMemberName;
 436
 437         Creator(final ShardIdentifier name, final Map<String, String> peerAddresses,
 438                 final DatastoreContext datastoreContext, final SchemaContext schemaContext,
 439                 final String localMemberName) {
 440             super(name, peerAddresses, datastoreContext, schemaContext);
 441             this.localMemberName = localMemberName;
 442         }
 443
 444         @Override
 445         public Shard create() throws Exception {
 446             return new EntityOwnershipShard(name, peerAddresses, datastoreContext, schemaContext, localMemberName);
 447         }
 448     }
 449
 450     private static interface EntityWalker {
 451         void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode);
 452     }
 453 }