Maintain EntityOwnershipStatistics 59/28259/7
authorMoiz Raja <moraja@cisco.com>
Sun, 11 Oct 2015 18:56:25 +0000 (11:56 -0700)
committerTom Pantelis <tpanteli@brocade.com>
Wed, 28 Oct 2015 13:21:57 +0000 (13:21 +0000)
Implementing a LoadBalancing entity owner selection
strategy depends on our ability to find the load on
specific candidates. The EntityOwnershipStatistics collects
this information and provides query methods to access
ownership counts for candidates.

Change-Id: I7e812b15e8fb21e3be1aed10384600b9acb8bf20
Signed-off-by: Moiz Raja <moraja@cisco.com>
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java [new file with mode: 0644]

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java
new file mode 100644 (file)
index 0000000..8392e12
--- /dev/null
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME;
+import org.opendaylight.controller.cluster.datastore.ShardDataTree;
+import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
+
+public abstract class AbstractEntityOwnerChangeListener  implements DOMDataTreeChangeListener {
+
+    void init(ShardDataTree shardDataTree) {
+        shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH).
+                node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this);
+    }
+
+    protected String extractOwner(LeafNode<?> ownerLeaf) {
+        Object value = ownerLeaf.getValue();
+        return value != null ? value.toString() : null;
+    }
+
+}
index cc0b8a318ddc29778cf17b82da2a78adb0237887..d10bc2158ae9824a5f6946d62d055da27895f35e 100644 (file)
@@ -7,17 +7,11 @@
  */
 package org.opendaylight.controller.cluster.datastore.entityownership;
 
-import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
-import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME;
 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.createEntity;
 import com.google.common.base.Objects;
 import com.google.common.base.Optional;
 import java.util.Collection;
-import org.opendaylight.controller.cluster.datastore.ShardDataTree;
 import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
-import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType;
-import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
@@ -30,7 +24,7 @@ import org.slf4j.LoggerFactory;
  *
  * @author Thomas Pantelis
  */
-class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
+class EntityOwnerChangeListener extends AbstractEntityOwnerChangeListener {
     private static final Logger LOG = LoggerFactory.getLogger(EntityOwnerChangeListener.class);
 
     private final String localMemberName;
@@ -41,11 +35,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
         this.listenerSupport = listenerSupport;
     }
 
-    void init(ShardDataTree shardDataTree) {
-        shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH).
-                node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this);
-    }
-
     @Override
     public void onDataTreeChanged(Collection<DataTreeCandidate> changes) {
         for(DataTreeCandidate change: changes) {
@@ -79,11 +68,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
         }
     }
 
-    private String extractOwner(LeafNode<?> ownerLeaf) {
-        Object value = ownerLeaf.getValue();
-        return value != null ? value.toString() : null;
-    }
-
     private String logId() {
         return listenerSupport.getLogId();
     }
index b98255635a1029817e45d708d630b1f9a290d299..2741ab772c5f56f531620c9bc0b3168f9baed8e2 100644 (file)
@@ -82,6 +82,7 @@ class EntityOwnershipShard extends Shard {
     private final Map<String, String> peerIdToMemberNames = new HashMap<>();
     private EntityOwnerSelectionStrategyConfig strategyConfig;
     private Map<YangInstanceIdentifier, Cancellable> entityToScheduledOwnershipTask = new HashMap<>();
+    private final EntityOwnershipStatistics entityOwnershipStatistics;
 
     private static DatastoreContext noPersistenceDatastoreContext(DatastoreContext datastoreContext) {
         return DatastoreContext.newBuilderFrom(datastoreContext).persistent(false).build();
@@ -95,6 +96,8 @@ class EntityOwnershipShard extends Shard {
         this.commitCoordinator = new EntityOwnershipShardCommitCoordinator(localMemberName, LOG);
         this.listenerSupport = new EntityOwnershipListenerSupport(getContext(), persistenceId());
         this.strategyConfig = strategyConfig;
+        this.entityOwnershipStatistics = new EntityOwnershipStatistics();
+        this.entityOwnershipStatistics.init(getDataStore());
 
         for(String peerId: peerAddresses.keySet()) {
             ShardIdentifier shardId = ShardIdentifier.builder().fromShardIdString(peerId).build();
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java
new file mode 100644 (file)
index 0000000..3ea4362
--- /dev/null
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityTypeFromEntityPath;
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
+import com.romix.scala.collection.concurrent.TrieMap;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidateNode;
+
+/**
+ * EntityOwnershipStatistics is a utility class that keeps track of ownership statistics for the candidates and
+ * caches it for quick count queries.
+ * <p>
+ * While the entity ownership model does maintain the information about which entity is owned by which candidate
+ * finding out how many entities of a given type are owned by a given candidate is not an efficient query.
+ */
+class EntityOwnershipStatistics extends AbstractEntityOwnerChangeListener {
+
+    private TrieMap<String, TrieMap<String, Long>> statistics = new TrieMap<>();
+
+    EntityOwnershipStatistics(){
+    }
+
+    @Override
+    public void onDataTreeChanged(@Nonnull Collection<DataTreeCandidate> changes) {
+        for (DataTreeCandidate change : changes) {
+            DataTreeCandidateNode changeRoot = change.getRootNode();
+            LeafNode<?> ownerLeaf = (LeafNode<?>) changeRoot.getDataAfter().get();
+            String entityType = entityTypeFromEntityPath(change.getRootPath());
+            String newOwner = extractOwner(ownerLeaf);
+            if(!Strings.isNullOrEmpty(newOwner)) {
+                updateStatistics(entityType, newOwner, 1);
+            }
+
+            Optional<NormalizedNode<?, ?>> dataBefore = changeRoot.getDataBefore();
+            if (dataBefore.isPresent()) {
+                String origOwner = extractOwner((LeafNode<?>) changeRoot.getDataBefore().get());
+                if(!Strings.isNullOrEmpty(origOwner)) {
+                    updateStatistics(entityType, origOwner, -1);
+                }
+            }
+        }
+    }
+
+    Map<String, Map<String, Long>> all() {
+        Map<String, Map<String, Long>> snapshot = new HashMap<>();
+        for (String entityType : statistics.readOnlySnapshot().keySet()) {
+            snapshot.put(entityType, byEntityType(entityType));
+        }
+        return snapshot;
+    }
+
+    Map<String, Long> byEntityType(String entityType){
+        return statistics.get(entityType).readOnlySnapshot();
+    }
+
+    private void updateStatistics(String entityType, String candidateName, long count){
+        Map<String, Long> m = statistics.get(entityType);
+        if(m == null){
+            m = new TrieMap<>();
+            m.put(candidateName, count);
+            statistics.put(entityType, m);
+        } else {
+            Long candidateOwnedEntities = m.get(candidateName);
+            if(candidateOwnedEntities == null){
+                m.put(candidateName, count);
+            } else {
+                m.put(candidateName, candidateOwnedEntities + count);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java
new file mode 100644 (file)
index 0000000..8c0d432
--- /dev/null
@@ -0,0 +1,132 @@
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.junit.Assert.assertEquals;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityEntryWithOwner;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityOwnersWithCandidate;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.opendaylight.controller.cluster.datastore.AbstractActorTest;
+import org.opendaylight.controller.cluster.datastore.ShardDataTree;
+import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException;
+
+public class EntityOwnershipStatisticsTest extends AbstractActorTest {
+    private static final String LOCAL_MEMBER_NAME = "member-1";
+    private static final String REMOTE_MEMBER_NAME1 = "member-2";
+    private static final String REMOTE_MEMBER_NAME2 = "member-3";
+    private static final String ENTITY_TYPE = "test";
+    private static final YangInstanceIdentifier ENTITY_ID1 =
+            YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity1"));
+    private static final YangInstanceIdentifier ENTITY_ID2 =
+            YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity2"));
+
+    private final ShardDataTree shardDataTree = new ShardDataTree(SchemaContextHelper.entityOwners());
+    private EntityOwnershipStatistics ownershipStatistics;
+
+    @Before
+    public void setup() {
+        ownershipStatistics = new EntityOwnershipStatistics();
+        ownershipStatistics.init(shardDataTree);
+    }
+
+    @Test
+    public void testOnDataTreeChanged() throws Exception {
+        Map<String, Map<String, Long>> statistics = null;
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME));
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID2, LOCAL_MEMBER_NAME));
+
+        // Write local member as owner for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME));
+        assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L);
+
+        // Add remote member 1 as candidate for entity 1 - ownershipStatistics support should not get notified
+
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, REMOTE_MEMBER_NAME1));
+        assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L);
+
+        // Change owner to remote member 1 for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME1));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L);
+
+        // Change owner to remote member 2 for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME2));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 1L);
+
+        // Clear the owner for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, ""));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to the local member for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to remote member 1 for entity 2
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, REMOTE_MEMBER_NAME1));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to the local member for entity 2
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Write local member owner for entity 2 again - expect no change
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Clear the owner for entity 2
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Clear the owner for entity 2 again - expect no change
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+    }
+
+    private void assertStatistics(Map<String, Map<String, Long>> statistics, String memberName, long val) {
+        assertEquals(val, statistics.get(ENTITY_TYPE).get(memberName).longValue());
+    }
+
+    private void writeNode(YangInstanceIdentifier path, NormalizedNode<?, ?> node) throws DataValidationFailedException {
+        AbstractEntityOwnershipTest.writeNode(path, node, shardDataTree);
+    }
+}
\ No newline at end of file