From ac3065a7d9b6666aa51c1507d9e8143a0cb4db07 Mon Sep 17 00:00:00 2001 From: Moiz Raja Date: Sun, 11 Oct 2015 11:56:25 -0700 Subject: [PATCH] Maintain EntityOwnershipStatistics Implementing a LoadBalancing entity owner selection strategy depends on our ability to find the load on specific candidates. The EntityOwnershipStatistics collects this information and provides query methods to access ownership counts for candidates. Change-Id: I7e812b15e8fb21e3be1aed10384600b9acb8bf20 Signed-off-by: Moiz Raja --- .../AbstractEntityOwnerChangeListener.java | 31 ++++ .../EntityOwnerChangeListener.java | 18 +-- .../entityownership/EntityOwnershipShard.java | 3 + .../EntityOwnershipStatistics.java | 86 +++++++++++ .../EntityOwnershipStatisticsTest.java | 140 ++++++++++++++++++ 5 files changed, 261 insertions(+), 17 deletions(-) create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java create mode 100644 opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java new file mode 100644 index 0000000000..b57f3c3246 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/AbstractEntityOwnerChangeListener.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.datastore.entityownership; + +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH; +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME; +import org.opendaylight.controller.cluster.datastore.ShardDataTree; +import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.schema.LeafNode; + +public abstract class AbstractEntityOwnerChangeListener implements DOMDataTreeChangeListener { + + void init(ShardDataTree shardDataTree) { + shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH). + node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this); + } + + protected static String extractOwner(LeafNode ownerLeaf) { + Object value = ownerLeaf.getValue(); + return value != null ? value.toString() : null; + } + +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java index bea0caa7c6..d10bc2158a 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnerChangeListener.java @@ -7,17 +7,11 @@ */ package org.opendaylight.controller.cluster.datastore.entityownership; -import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH; -import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_QNAME; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.createEntity; import com.google.common.base.Objects; import com.google.common.base.Optional; import java.util.Collection; -import org.opendaylight.controller.cluster.datastore.ShardDataTree; import org.opendaylight.controller.md.sal.common.api.clustering.Entity; -import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener; -import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.EntityType; -import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.schema.LeafNode; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; @@ -30,7 +24,7 @@ import org.slf4j.LoggerFactory; * * @author Thomas Pantelis */ -class EntityOwnerChangeListener implements DOMDataTreeChangeListener { +class EntityOwnerChangeListener extends AbstractEntityOwnerChangeListener { private static final Logger LOG = LoggerFactory.getLogger(EntityOwnerChangeListener.class); private final String localMemberName; @@ -41,11 +35,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener { this.listenerSupport = listenerSupport; } - void init(ShardDataTree shardDataTree) { - shardDataTree.registerTreeChangeListener(YangInstanceIdentifier.builder(ENTITY_OWNERS_PATH). - node(EntityType.QNAME).node(EntityType.QNAME).node(ENTITY_QNAME).node(ENTITY_QNAME).node(EntityOwnersModel.ENTITY_OWNER_QNAME).build(), this); - } - @Override public void onDataTreeChanged(Collection changes) { for(DataTreeCandidate change: changes) { @@ -79,11 +68,6 @@ class EntityOwnerChangeListener implements DOMDataTreeChangeListener { } } - private static String extractOwner(LeafNode ownerLeaf) { - Object value = ownerLeaf.getValue(); - return value != null ? value.toString() : null; - } - private String logId() { return listenerSupport.getLogId(); } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java index ba7512830e..60fab39b8e 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java @@ -80,6 +80,7 @@ class EntityOwnershipShard extends Shard { private final Map peerIdToMemberNames = new HashMap<>(); private final EntityOwnerSelectionStrategyConfig strategyConfig; private final Map entityToScheduledOwnershipTask = new HashMap<>(); + private final EntityOwnershipStatistics entityOwnershipStatistics; private static DatastoreContext noPersistenceDatastoreContext(DatastoreContext datastoreContext) { return DatastoreContext.newBuilderFrom(datastoreContext).persistent(false).build(); @@ -91,6 +92,8 @@ class EntityOwnershipShard extends Shard { this.commitCoordinator = new EntityOwnershipShardCommitCoordinator(builder.localMemberName, LOG); this.listenerSupport = new EntityOwnershipListenerSupport(getContext(), persistenceId()); this.strategyConfig = builder.ownerSelectionStrategyConfig; + this.entityOwnershipStatistics = new EntityOwnershipStatistics(); + this.entityOwnershipStatistics.init(getDataStore()); for(String peerId: getRaftActorContext().getPeerIds()) { ShardIdentifier shardId = ShardIdentifier.builder().fromShardIdString(peerId).build(); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java new file mode 100644 index 0000000000..3ea4362ac9 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatistics.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.datastore.entityownership; + +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityTypeFromEntityPath; +import com.google.common.base.Optional; +import com.google.common.base.Strings; +import com.romix.scala.collection.concurrent.TrieMap; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nonnull; +import org.opendaylight.yangtools.yang.data.api.schema.LeafNode; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidateNode; + +/** + * EntityOwnershipStatistics is a utility class that keeps track of ownership statistics for the candidates and + * caches it for quick count queries. + *

+ * While the entity ownership model does maintain the information about which entity is owned by which candidate + * finding out how many entities of a given type are owned by a given candidate is not an efficient query. + */ +class EntityOwnershipStatistics extends AbstractEntityOwnerChangeListener { + + private TrieMap> statistics = new TrieMap<>(); + + EntityOwnershipStatistics(){ + } + + @Override + public void onDataTreeChanged(@Nonnull Collection changes) { + for (DataTreeCandidate change : changes) { + DataTreeCandidateNode changeRoot = change.getRootNode(); + LeafNode ownerLeaf = (LeafNode) changeRoot.getDataAfter().get(); + String entityType = entityTypeFromEntityPath(change.getRootPath()); + String newOwner = extractOwner(ownerLeaf); + if(!Strings.isNullOrEmpty(newOwner)) { + updateStatistics(entityType, newOwner, 1); + } + + Optional> dataBefore = changeRoot.getDataBefore(); + if (dataBefore.isPresent()) { + String origOwner = extractOwner((LeafNode) changeRoot.getDataBefore().get()); + if(!Strings.isNullOrEmpty(origOwner)) { + updateStatistics(entityType, origOwner, -1); + } + } + } + } + + Map> all() { + Map> snapshot = new HashMap<>(); + for (String entityType : statistics.readOnlySnapshot().keySet()) { + snapshot.put(entityType, byEntityType(entityType)); + } + return snapshot; + } + + Map byEntityType(String entityType){ + return statistics.get(entityType).readOnlySnapshot(); + } + + private void updateStatistics(String entityType, String candidateName, long count){ + Map m = statistics.get(entityType); + if(m == null){ + m = new TrieMap<>(); + m.put(candidateName, count); + statistics.put(entityType, m); + } else { + Long candidateOwnedEntities = m.get(candidateName); + if(candidateOwnedEntities == null){ + m.put(candidateName, count); + } else { + m.put(candidateName, candidateOwnedEntities + count); + } + } + } +} \ No newline at end of file diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java new file mode 100644 index 0000000000..7374de3698 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipStatisticsTest.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.controller.cluster.datastore.entityownership; + +import static org.junit.Assert.assertEquals; +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_OWNERS_PATH; +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityEntryWithOwner; +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityOwnersWithCandidate; +import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath; +import java.util.Map; +import org.junit.Before; +import org.junit.Test; +import org.opendaylight.controller.cluster.datastore.AbstractActorTest; +import org.opendaylight.controller.cluster.datastore.ShardDataTree; +import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException; + +public class EntityOwnershipStatisticsTest extends AbstractActorTest { + private static final String LOCAL_MEMBER_NAME = "member-1"; + private static final String REMOTE_MEMBER_NAME1 = "member-2"; + private static final String REMOTE_MEMBER_NAME2 = "member-3"; + private static final String ENTITY_TYPE = "test"; + private static final YangInstanceIdentifier ENTITY_ID1 = + YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity1")); + private static final YangInstanceIdentifier ENTITY_ID2 = + YangInstanceIdentifier.of(QName.create("test", "2015-08-14", "entity2")); + + private final ShardDataTree shardDataTree = new ShardDataTree(SchemaContextHelper.entityOwners()); + private EntityOwnershipStatistics ownershipStatistics; + + @Before + public void setup() { + ownershipStatistics = new EntityOwnershipStatistics(); + ownershipStatistics.init(shardDataTree); + } + + @Test + public void testOnDataTreeChanged() throws Exception { + Map> statistics = null; + writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, LOCAL_MEMBER_NAME)); + writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID2, LOCAL_MEMBER_NAME)); + + // Write local member as owner for entity 1 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME)); + assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L); + + // Add remote member 1 as candidate for entity 1 - ownershipStatistics support should not get notified + + writeNode(ENTITY_OWNERS_PATH, entityOwnersWithCandidate(ENTITY_TYPE, ENTITY_ID1, REMOTE_MEMBER_NAME1)); + assertStatistics(ownershipStatistics.all(), LOCAL_MEMBER_NAME, 1L); + + // Change owner to remote member 1 for entity 1 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME1)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L); + + // Change owner to remote member 2 for entity 1 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, REMOTE_MEMBER_NAME2)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 1L); + + // Clear the owner for entity 1 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, "")); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Change owner to the local member for entity 1 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID1), entityEntryWithOwner(ENTITY_ID1, LOCAL_MEMBER_NAME)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Change owner to remote member 1 for entity 2 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, REMOTE_MEMBER_NAME1)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 1L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Change owner to the local member for entity 2 + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Write local member owner for entity 2 again - expect no change + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, LOCAL_MEMBER_NAME)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 2L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Clear the owner for entity 2 + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + // Clear the owner for entity 2 again - expect no change + + writeNode(entityPath(ENTITY_TYPE, ENTITY_ID2), entityEntryWithOwner(ENTITY_ID2, null)); + statistics = ownershipStatistics.all(); + assertStatistics(statistics, LOCAL_MEMBER_NAME, 1L); + assertStatistics(statistics, REMOTE_MEMBER_NAME1, 0L); + assertStatistics(statistics, REMOTE_MEMBER_NAME2, 0L); + + } + + private void assertStatistics(Map> statistics, String memberName, long val) { + assertEquals(val, statistics.get(ENTITY_TYPE).get(memberName).longValue()); + } + + private void writeNode(YangInstanceIdentifier path, NormalizedNode node) throws DataValidationFailedException { + AbstractEntityOwnershipTest.writeNode(path, node, shardDataTree); + } +} \ No newline at end of file -- 2.36.6