Maintain EntityOwnershipStatistics 06/29406/6
authorMoiz Raja <moraja@cisco.com>
Sun, 11 Oct 2015 18:56:25 +0000 (11:56 -0700)
committerTom Pantelis <tpanteli@brocade.com>
Fri, 13 Nov 2015 00:39:57 +0000 (19:39 -0500)
Implementing a LoadBalancing entity owner selection
strategy depends on our ability to find the load on
specific candidates. The EntityOwnershipStatistics collects
this information and provides query methods to access
ownership counts for candidates.

Change-Id: I7e812b15e8fb21e3be1aed10384600b9acb8bf20
Signed-off-by: Moiz Raja <moraja@cisco.com>
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java [new file with mode: 0644]

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java
new file mode 100644 (file)
index 0000000..b57f3c3
--- /dev/null
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME;
+import org.opendaylight.controller.cluster.datastore.ShardDataTree;
+import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
+
+public abstract class AbstractEntityOwnerChangeListener  implements DOMDataTreeChangeListener {
+
+    void init(ShardDataTree shardDataTree) {
+        shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH).
+                node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this);
+    }
+
+    protected static String extractOwner(LeafNode<?> ownerLeaf) {
+        Object value = ownerLeaf.getValue();
+        return value != null ? value.toString() : null;
+    }
+
+}
index bea0caa7c666ed2ae158a6e130cf32c04e703730..d10bc2158ae9824a5f6946d62d055da27895f35e 100644 (file)
@@ -7,17 +7,11 @@
  */
 package org.opendaylight.controller.cluster.datastore.entityownership;
 
-import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
-import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME;
 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.createEntity;
 import com.google.common.base.Objects;
 import com.google.common.base.Optional;
 import java.util.Collection;
-import org.opendaylight.controller.cluster.datastore.ShardDataTree;
 import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
-import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType;
-import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
@@ -30,7 +24,7 @@ import org.slf4j.LoggerFactory;
  *
  * @author Thomas Pantelis
  */
-class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
+class EntityOwnerChangeListener extends AbstractEntityOwnerChangeListener {
     private static final Logger LOG = LoggerFactory.getLogger(EntityOwnerChangeListener.class);
 
     private final String localMemberName;
@@ -41,11 +35,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
         this.listenerSupport = listenerSupport;
     }
 
-    void init(ShardDataTree shardDataTree) {
-        shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH).
-                node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this);
-    }
-
     @Override
     public void onDataTreeChanged(Collection<DataTreeCandidate> changes) {
         for(DataTreeCandidate change: changes) {
@@ -79,11 +68,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener {
         }
     }
 
-    private static String extractOwner(LeafNode<?> ownerLeaf) {
-        Object value = ownerLeaf.getValue();
-        return value != null ? value.toString() : null;
-    }
-
     private String logId() {
         return listenerSupport.getLogId();
     }
index ba7512830ea1b8b65a5343946c6098095bf363f0..60fab39b8e856f98bfbe47f3dc44e43d51bedf82 100644 (file)
@@ -80,6 +80,7 @@ class EntityOwnershipShard extends Shard {
     private final Map<String, String> peerIdToMemberNames = new HashMap<>();
     private final EntityOwnerSelectionStrategyConfig strategyConfig;
     private final Map<YangInstanceIdentifier, Cancellable> entityToScheduledOwnershipTask = new HashMap<>();
+    private final EntityOwnershipStatistics entityOwnershipStatistics;
 
     private static DatastoreContext noPersistenceDatastoreContext(DatastoreContext datastoreContext) {
         return DatastoreContext.newBuilderFrom(datastoreContext).persistent(false).build();
@@ -91,6 +92,8 @@ class EntityOwnershipShard extends Shard {
         this.commitCoordinator = new EntityOwnershipShardCommitCoordinator(builder.localMemberName, LOG);
         this.listenerSupport = new EntityOwnershipListenerSupport(getContext(), persistenceId());
         this.strategyConfig = builder.ownerSelectionStrategyConfig;
+        this.entityOwnershipStatistics = new EntityOwnershipStatistics();
+        this.entityOwnershipStatistics.init(getDataStore());
 
         for(String peerId: getRaftActorContext().getPeerIds()) {
             ShardIdentifier shardId = ShardIdentifier.builder().fromShardIdString(peerId).build();
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java
new file mode 100644 (file)
index 0000000..3ea4362
--- /dev/null
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityTypeFromEntityPath;
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
+import com.romix.scala.collection.concurrent.TrieMap;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.Nonnull;
+import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidateNode;
+
+/**
+ * EntityOwnershipStatistics is a utility class that keeps track of ownership statistics for the candidates and
+ * caches it for quick count queries.
+ * <p>
+ * While the entity ownership model does maintain the information about which entity is owned by which candidate
+ * finding out how many entities of a given type are owned by a given candidate is not an efficient query.
+ */
+class EntityOwnershipStatistics extends AbstractEntityOwnerChangeListener {
+
+    private TrieMap<String, TrieMap<String, Long>> statistics = new TrieMap<>();
+
+    EntityOwnershipStatistics(){
+    }
+
+    @Override
+    public void onDataTreeChanged(@Nonnull Collection<DataTreeCandidate> changes) {
+        for (DataTreeCandidate change : changes) {
+            DataTreeCandidateNode changeRoot = change.getRootNode();
+            LeafNode<?> ownerLeaf = (LeafNode<?>) changeRoot.getDataAfter().get();
+            String entityType = entityTypeFromEntityPath(change.getRootPath());
+            String newOwner = extractOwner(ownerLeaf);
+            if(!Strings.isNullOrEmpty(newOwner)) {
+                updateStatistics(entityType, newOwner, 1);
+            }
+
+            Optional<NormalizedNode<?, ?>> dataBefore = changeRoot.getDataBefore();
+            if (dataBefore.isPresent()) {
+                String origOwner = extractOwner((LeafNode<?>) changeRoot.getDataBefore().get());
+                if(!Strings.isNullOrEmpty(origOwner)) {
+                    updateStatistics(entityType, origOwner, -1);
+                }
+            }
+        }
+    }
+
+    Map<String, Map<String, Long>> all() {
+        Map<String, Map<String, Long>> snapshot = new HashMap<>();
+        for (String entityType : statistics.readOnlySnapshot().keySet()) {
+            snapshot.put(entityType, byEntityType(entityType));
+        }
+        return snapshot;
+    }
+
+    Map<String, Long> byEntityType(String entityType){
+        return statistics.get(entityType).readOnlySnapshot();
+    }
+
+    private void updateStatistics(String entityType, String candidateName, long count){
+        Map<String, Long> m = statistics.get(entityType);
+        if(m == null){
+            m = new TrieMap<>();
+            m.put(candidateName, count);
+            statistics.put(entityType, m);
+        } else {
+            Long candidateOwnedEntities = m.get(candidateName);
+            if(candidateOwnedEntities == null){
+                m.put(candidateName, count);
+            } else {
+                m.put(candidateName, candidateOwnedEntities + count);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java
new file mode 100644 (file)
index 0000000..7374de3
--- /dev/null
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.controller.cluster.datastore.entityownership;
+
+import static org.junit.Assert.assertEquals;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityEntryWithOwner;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityOwnersWithCandidate;
+import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.opendaylight.controller.cluster.datastore.AbstractActorTest;
+import org.opendaylight.controller.cluster.datastore.ShardDataTree;
+import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException;
+
+public class EntityOwnershipStatisticsTest extends AbstractActorTest {
+    private static final String LOCAL_MEMBER_NAME = "member-1";
+    private static final String REMOTE_MEMBER_NAME1 = "member-2";
+    private static final String REMOTE_MEMBER_NAME2 = "member-3";
+    private static final String ENTITY_TYPE = "test";
+    private static final YangInstanceIdentifier ENTITY_ID1 =
+            YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity1"));
+    private static final YangInstanceIdentifier ENTITY_ID2 =
+            YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity2"));
+
+    private final ShardDataTree shardDataTree = new ShardDataTree(SchemaContextHelper.entityOwners());
+    private EntityOwnershipStatistics ownershipStatistics;
+
+    @Before
+    public void setup() {
+        ownershipStatistics = new EntityOwnershipStatistics();
+        ownershipStatistics.init(shardDataTree);
+    }
+
+    @Test
+    public void testOnDataTreeChanged() throws Exception {
+        Map<String, Map<String, Long>> statistics = null;
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME));
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID2, LOCAL_MEMBER_NAME));
+
+        // Write local member as owner for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME));
+        assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L);
+
+        // Add remote member 1 as candidate for entity 1 - ownershipStatistics support should not get notified
+
+        writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, REMOTE_MEMBER_NAME1));
+        assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L);
+
+        // Change owner to remote member 1 for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME1));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L);
+
+        // Change owner to remote member 2 for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME2));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 1L);
+
+        // Clear the owner for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, ""));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to the local member for entity 1
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to remote member 1 for entity 2
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, REMOTE_MEMBER_NAME1));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Change owner to the local member for entity 2
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Write local member owner for entity 2 again - expect no change
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Clear the owner for entity 2
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+        // Clear the owner for entity 2 again - expect no change
+
+        writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null));
+        statistics = ownershipStatistics.all();
+        assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L);
+        assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L);
+
+    }
+
+    private void assertStatistics(Map<String, Map<String, Long>> statistics, String memberName, long val) {
+        assertEquals(val, statistics.get(ENTITY_TYPE).get(memberName).longValue());
+    }
+
+    private void writeNode(YangInstanceIdentifier path, NormalizedNode<?, ?> node) throws DataValidationFailedException {
+        AbstractEntityOwnershipTest.writeNode(path, node, shardDataTree);
+    }
+}
\ No newline at end of file