From: Anil Vishnoi Date: Sat, 3 Jan 2015 20:18:00 +0000 (+0530) Subject: Bug 2551 - Statistics collection of random node fails when large number if switches... X-Git-Tag: release/lithium~733^2 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=a5899521495f3a3d089f42ed026bd5a6fea62411 Bug 2551 - Statistics collection of random node fails when large number if switches disconnects from controller. When statistics manager is collecting stats and large number of switches disconnects from controller ,some time it causes OptimisticsLockFailedException while processing the existing multipart reply present in dataStoreOperQueue. Exception occures because it tries to write data to the Node that doesn't existing in the data store. While handling OptimisticsLockFailedException, statistics manager flush existing data store related operation task from dataStoreOperQueue. This queue also holds Operational task for cleaning up internal data of the disconnected nodes and because it flushes the queue, those operational task don't get executed and statistics manager assumes that those nodes are yet connected to the controller. When all the switches connects back with different mode (e.g previsouly Openflow13 but reconnects in OpenFlow10 mode), statistics collection of those node fails. Change-Id: I50a44dfd20f90e3179bcd15bde67247da6565af7 Signed-off-by: Anil Vishnoi --- diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/StatisticsManager.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/StatisticsManager.java index 751a68965d..6124bdf642 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/StatisticsManager.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/StatisticsManager.java @@ -24,6 +24,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.transaction.rev131103. import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.port.rev130925.queues.Queue; import org.opendaylight.yang.gen.v1.urn.opendaylight.group.statistics.rev131111.OpendaylightGroupStatisticsListener; import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group; +import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; import org.opendaylight.yang.gen.v1.urn.opendaylight.meter.statistics.rev131111.OpendaylightMeterStatisticsListener; import org.opendaylight.yang.gen.v1.urn.opendaylight.port.statistics.rev131214.OpendaylightPortStatisticsListener; @@ -53,14 +54,47 @@ public interface StatisticsManager extends AutoCloseable, TransactionChainListen * Internal {@link TransactionChainListener} joining all DS commits * to Set of chained changes for prevent often DataStore touches. */ - public interface StatDataStoreOperation { + public abstract class StatDataStoreOperation { + public enum StatsManagerOperationType { + /** + * Operation will carry out work related to new node addition / + * update + */ + NODE_UPDATE, + /** + * Operation will carry out work related to node removal + */ + NODE_REMOVAL, + /** + * Operation will commit data to the operational data store + */ + DATA_COMMIT_OPER_DS + } + + private NodeId nodeId; + private StatsManagerOperationType operationType = StatsManagerOperationType.DATA_COMMIT_OPER_DS; + + public StatDataStoreOperation(final StatsManagerOperationType operType, final NodeId id){ + if(operType != null){ + operationType = operType; + } + nodeId = id; + } + + public final StatsManagerOperationType getType() { + return operationType; + } + + public final NodeId getNodeId(){ + return nodeId; + } /** - * Apply all read / write (put|merge) operation - * for DataStore + * Apply all read / write (put|merge) operation for DataStore + * * @param {@link ReadWriteTransaction} tx */ - void applyOperation(ReadWriteTransaction tx); + public abstract void applyOperation(ReadWriteTransaction tx); } diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatListenCommitFlow.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatListenCommitFlow.java index e17c45dc76..49fe3bbefd 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatListenCommitFlow.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatListenCommitFlow.java @@ -25,6 +25,7 @@ import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.controller.md.statistics.manager.StatRpcMsgManager.TransactionCacheContainer; import org.opendaylight.controller.md.statistics.manager.StatisticsManager; import org.opendaylight.controller.md.statistics.manager.StatisticsManager.StatDataStoreOperation; +import org.opendaylight.controller.md.statistics.manager.StatisticsManager.StatDataStoreOperation.StatsManagerOperationType; import org.opendaylight.controller.md.statistics.manager.impl.helper.FlowComparator; import org.opendaylight.controller.sal.binding.api.NotificationProviderService; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; @@ -120,7 +121,7 @@ public class StatListenCommitFlow extends StatAbstractListenCommit> txContainer = getTransactionCacheContainer(transId, nodeId); @@ -218,6 +219,7 @@ public class StatListenCommitFlow extends StatAbstractListenCommit nodeIdent = InstanceIdentifier @@ -157,7 +158,7 @@ public class StatListenCommitGroup extends StatAbstractListenCommit nodeIdent = InstanceIdentifier.create(Nodes.class) .child(Node.class, new NodeKey(nodeId)); /* Don't block RPC Notification thread */ - manager.enqueue(new StatDataStoreOperation() { + manager.enqueue(new StatDataStoreOperation(StatsManagerOperationType.DATA_COMMIT_OPER_DS,nodeId) { @Override public void applyOperation(final ReadWriteTransaction trans) { final Optional> txContainer = getTransactionCacheContainer(transId, nodeId); diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatNotifyCommitTable.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatNotifyCommitTable.java index 53bca87034..2d730645ac 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatNotifyCommitTable.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatNotifyCommitTable.java @@ -17,6 +17,7 @@ import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.controller.md.statistics.manager.StatRpcMsgManager.TransactionCacheContainer; import org.opendaylight.controller.md.statistics.manager.StatisticsManager; import org.opendaylight.controller.md.statistics.manager.StatisticsManager.StatDataStoreOperation; +import org.opendaylight.controller.md.statistics.manager.StatisticsManager.StatDataStoreOperation.StatsManagerOperationType; import org.opendaylight.controller.sal.binding.api.NotificationProviderService; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table; @@ -81,7 +82,7 @@ public class StatNotifyCommitTable extends StatAbstractNotifyCommit tableStats = new ArrayList(10); diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatisticsManagerImpl.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatisticsManagerImpl.java index 1d03e38c16..437c92f6a0 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatisticsManagerImpl.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/impl/StatisticsManagerImpl.java @@ -29,6 +29,7 @@ import org.opendaylight.controller.md.statistics.manager.StatPermCollector; import org.opendaylight.controller.md.statistics.manager.StatPermCollector.StatCapabTypes; import org.opendaylight.controller.md.statistics.manager.StatRpcMsgManager; import org.opendaylight.controller.md.statistics.manager.StatisticsManager; +import org.opendaylight.controller.md.statistics.manager.StatisticsManager.StatDataStoreOperation.StatsManagerOperationType; import org.opendaylight.controller.sal.binding.api.NotificationProviderService; import org.opendaylight.controller.sal.binding.api.RpcConsumerRegistry; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.Meter; @@ -223,7 +224,18 @@ public class StatisticsManagerImpl implements StatisticsManager, Runnable { private synchronized void cleanDataStoreOperQueue() { // Drain all events, making sure any blocked threads are unblocked while (! dataStoreOperQueue.isEmpty()) { - dataStoreOperQueue.poll(); + StatDataStoreOperation op = dataStoreOperQueue.poll(); + + // Execute the node removal clean up operation if queued in the + // operational queue. + if (op.getType() == StatsManagerOperationType.NODE_REMOVAL) { + try { + LOG.debug("Node {} disconnected. Cleaning internal data.",op.getNodeId()); + op.applyOperation(null); + } catch (final Exception ex) { + LOG.warn("Unhandled exception while cleaning up internal data of node [{}]",op.getNodeId()); + } + } } }