From aae447eb2ce6272e1bfd2517a6493bf2ea40ed7a Mon Sep 17 00:00:00 2001 From: Anil Vishnoi Date: Wed, 26 Mar 2014 03:19:09 +0530 Subject: [PATCH] Statistics-Manager - Performance Improvement 1) Introduced statistics request scheduler, to schedule request based on the current transaction load on MD-SAL DataStore. Each node submit all individual statistics request to schedular for execution 2) Send statistics request if there is no MD-SAL trasaction pending. It just keep tracks of the MD-SAL trasaction triggered by statistics-manager 3) Removal of stale statistics is now done based on counter rather then time values. Time based removal will break in case of clustered environment. 4) Code clean up Change-Id: Ie7522d0c60f2c7051dbfcdf9a6657843ef4da743 Signed-off-by: Anil Vishnoi --- .../AbstractListeningStatsTracker.java | 4 +- .../manager/AbstractStatsTracker.java | 25 ++-- .../statistics/manager/FlowStatsTracker.java | 24 +++- .../manager/FlowTableStatsTracker.java | 5 +- .../manager/GroupDescStatsTracker.java | 5 +- .../statistics/manager/GroupStatsTracker.java | 5 +- .../manager/MeterConfigStatsTracker.java | 5 +- .../statistics/manager/MeterStatsTracker.java | 5 +- .../manager/NodeConnectorStatsTracker.java | 5 +- .../manager/NodeStatisticsHandler.java | 86 ++++++------- .../statistics/manager/QueueStatsTracker.java | 5 +- .../manager/StatisticsProvider.java | 8 +- .../manager/StatisticsRequestScheduler.java | 118 ++++++++++++++++++ 13 files changed, 230 insertions(+), 70 deletions(-) create mode 100644 opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/StatisticsRequestScheduler.java diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractListeningStatsTracker.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractListeningStatsTracker.java index 4a58579b13..167fb21ffd 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractListeningStatsTracker.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractListeningStatsTracker.java @@ -20,8 +20,8 @@ abstract class AbstractListeningStatsTracker extends AbstractStatsTracker< private static final Logger logger = LoggerFactory.getLogger(AbstractListeningStatsTracker.class); private ListenerRegistration reg; - protected AbstractListeningStatsTracker(FlowCapableContext context, long lifetimeNanos) { - super(context, lifetimeNanos); + protected AbstractListeningStatsTracker(FlowCapableContext context) { + super(context); } protected abstract InstanceIdentifier listenPath(); diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractStatsTracker.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractStatsTracker.java index c29b6a7730..e922656d91 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractStatsTracker.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/AbstractStatsTracker.java @@ -32,6 +32,9 @@ import com.google.common.util.concurrent.JdkFutureAdapters; abstract class AbstractStatsTracker { private static final Logger logger = LoggerFactory.getLogger(AbstractStatsTracker.class); + + private static final int WAIT_FOR_REQUEST_CYCLE = 2; + private final FutureCallback> callback = new FutureCallback>() { @Override @@ -62,11 +65,11 @@ abstract class AbstractStatsTracker { private final Map trackedItems = new HashMap<>(); private final FlowCapableContext context; - private final long lifetimeNanos; + private long requestCounter; - protected AbstractStatsTracker(final FlowCapableContext context, final long lifetimeNanos) { + protected AbstractStatsTracker(final FlowCapableContext context) { this.context = Preconditions.checkNotNull(context); - this.lifetimeNanos = lifetimeNanos; + this.requestCounter = 0; } protected final InstanceIdentifierBuilder getNodeIdentifierBuilder() { @@ -89,24 +92,32 @@ abstract class AbstractStatsTracker { return context.startDataModification(); } + public final synchronized void increaseRequestCounter(){ + this.requestCounter++; + } protected abstract void cleanupSingleStat(DataModificationTransaction trans, K item); protected abstract K updateSingleStat(DataModificationTransaction trans, I item); + public abstract void request(); public final synchronized void updateStats(List list) { - final Long expiryTime = System.nanoTime() + lifetimeNanos; + final DataModificationTransaction trans = startTransaction(); for (final I item : list) { - trackedItems.put(updateSingleStat(trans, item), expiryTime); + trackedItems.put(updateSingleStat(trans, item), requestCounter); } trans.commit(); } - public final synchronized void cleanup(final DataModificationTransaction trans, long now) { + /** + * Statistics will be cleaned up if not update in last two request cycles. + * @param trans + */ + public final synchronized void cleanup(final DataModificationTransaction trans) { for (Iterator> it = trackedItems.entrySet().iterator();it.hasNext();){ Entry e = it.next(); - if (now > e.getValue()) { + if (requestCounter >= e.getValue()+WAIT_FOR_REQUEST_CYCLE) { cleanupSingleStat(trans, e.getKey()); it.remove(); } diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowStatsTracker.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowStatsTracker.java index 90ddc28acd..06d6e82112 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowStatsTracker.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowStatsTracker.java @@ -7,6 +7,7 @@ */ package org.opendaylight.controller.md.statistics.manager; +import java.util.Collection; import java.util.Map.Entry; import org.opendaylight.controller.md.sal.common.api.data.DataChangeEvent; @@ -37,12 +38,17 @@ import org.slf4j.LoggerFactory; final class FlowStatsTracker extends AbstractListeningStatsTracker { private static final Logger logger = LoggerFactory.getLogger(FlowStatsTracker.class); private final OpendaylightFlowStatisticsService flowStatsService; + private FlowTableStatsTracker flowTableStats; private int unaccountedFlowsCounter = 1; - FlowStatsTracker(OpendaylightFlowStatisticsService flowStatsService, final FlowCapableContext context, long lifetimeNanos) { - super(context, lifetimeNanos); + FlowStatsTracker(OpendaylightFlowStatisticsService flowStatsService, final FlowCapableContext context) { + super(context); this.flowStatsService = flowStatsService; } + FlowStatsTracker(OpendaylightFlowStatisticsService flowStatsService, final FlowCapableContext context, FlowTableStatsTracker flowTableStats) { + this(flowStatsService, context); + this.flowTableStats = flowTableStats; + } @Override protected void cleanupSingleStat(DataModificationTransaction trans, FlowStatsEntry item) { @@ -203,6 +209,20 @@ final class FlowStatsTracker extends AbstractListeningStatsTracker tables = flowTableStats.getTables(); + logger.debug("Node {} supports {} table(s)", this.getNodeRef(), tables.size()); + for (final TableKey key : tables) { + logger.debug("Send aggregate stats request for flow table {} to node {}", key.getId(), this.getNodeRef()); + this.requestAggregateFlows(key); + } + + this.requestAllFlowsAllTables(); + + } public void requestAllFlowsAllTables() { if (flowStatsService != null) { final GetAllFlowsStatisticsFromAllFlowTablesInputBuilder input = new GetAllFlowsStatisticsFromAllFlowTablesInputBuilder(); diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowTableStatsTracker.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowTableStatsTracker.java index 3fe68c111a..a160f6d467 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowTableStatsTracker.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/FlowTableStatsTracker.java @@ -30,8 +30,8 @@ final class FlowTableStatsTracker extends AbstractStatsTracker tables = Collections.unmodifiableSet(privateTables); private final OpendaylightFlowTableStatisticsService flowTableStatsService; - FlowTableStatsTracker(OpendaylightFlowTableStatisticsService flowTableStatsService, final FlowCapableContext context, long lifetimeNanos) { - super(context, lifetimeNanos); + FlowTableStatsTracker(OpendaylightFlowTableStatisticsService flowTableStatsService, final FlowCapableContext context) { + super(context); this.flowTableStatsService = flowTableStatsService; } @@ -61,6 +61,7 @@ final class FlowTableStatsTracker extends AbstractStatsTracker targetNodeIdentifier; private final FlowStatsTracker flowStats; private final FlowTableStatsTracker flowTableStats; @@ -103,23 +103,25 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo final OpendaylightGroupStatisticsService groupStatsService, final OpendaylightMeterStatisticsService meterStatsService, final OpendaylightPortStatisticsService portStatsService, - final OpendaylightQueueStatisticsService queueStatsService) { + final OpendaylightQueueStatisticsService queueStatsService, + final StatisticsRequestScheduler srScheduler) { this.dps = Preconditions.checkNotNull(dps); this.targetNodeKey = Preconditions.checkNotNull(nodeKey); + this.srScheduler = Preconditions.checkNotNull(srScheduler); this.targetNodeIdentifier = InstanceIdentifier.builder(Nodes.class).child(Node.class, targetNodeKey).build(); this.targetNodeRef = new NodeRef(targetNodeIdentifier); final long lifetimeNanos = TimeUnit.MILLISECONDS.toNanos(STATS_COLLECTION_MILLIS * NUMBER_OF_WAIT_CYCLES); msgManager = new MultipartMessageManager(lifetimeNanos); - flowStats = new FlowStatsTracker(flowStatsService, this, lifetimeNanos); - flowTableStats = new FlowTableStatsTracker(flowTableStatsService, this, lifetimeNanos); - groupDescStats = new GroupDescStatsTracker(groupStatsService, this, lifetimeNanos); - groupStats = new GroupStatsTracker(groupStatsService, this, lifetimeNanos); - meterConfigStats = new MeterConfigStatsTracker(meterStatsService, this, lifetimeNanos); - meterStats = new MeterStatsTracker(meterStatsService, this, lifetimeNanos); - nodeConnectorStats = new NodeConnectorStatsTracker(portStatsService, this, lifetimeNanos); - queueStats = new QueueStatsTracker(queueStatsService, this, lifetimeNanos); + flowTableStats = new FlowTableStatsTracker(flowTableStatsService, this); + flowStats = new FlowStatsTracker(flowStatsService, this, flowTableStats); + groupDescStats = new GroupDescStatsTracker(groupStatsService, this); + groupStats = new GroupStatsTracker(groupStatsService, this); + meterConfigStats = new MeterConfigStatsTracker(meterStatsService, this); + meterStats = new MeterStatsTracker(meterStatsService, this); + nodeConnectorStats = new NodeConnectorStatsTracker(portStatsService, this); + queueStats = new QueueStatsTracker(queueStatsService, this); } public NodeKey getTargetNodeKey() { @@ -138,7 +140,9 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo @Override public DataModificationTransaction startDataModification() { - return dps.beginTransaction(); + DataModificationTransaction dmt = dps.beginTransaction(); + dmt.registerListener(this.srScheduler); + return dmt; } public synchronized void updateGroupDescStats(TransactionAware transaction, List list) { @@ -186,7 +190,7 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo public synchronized void updateAggregateFlowStats(TransactionAware transaction, AggregateFlowStatistics flowStats) { final Short tableId = msgManager.isExpectedTableTransaction(transaction); if (tableId != null) { - final DataModificationTransaction trans = dps.beginTransaction(); + final DataModificationTransaction trans = this.startDataModification(); InstanceIdentifier tableRef = InstanceIdentifier.builder(Nodes.class).child(Node.class, targetNodeKey) .augmentation(FlowCapableNode.class).child(Table.class, new TableKey(tableId)).toInstance(); @@ -214,7 +218,7 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo } public synchronized void updateGroupFeatures(GroupFeatures notification) { - final DataModificationTransaction trans = dps.beginTransaction(); + final DataModificationTransaction trans = this.startDataModification(); final NodeBuilder nodeData = new NodeBuilder(); nodeData.setKey(targetNodeKey); @@ -232,7 +236,7 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo } public synchronized void updateMeterFeatures(MeterFeatures features) { - final DataModificationTransaction trans = dps.beginTransaction(); + final DataModificationTransaction trans = this.startDataModification(); final NodeBuilder nodeData = new NodeBuilder(); nodeData.setKey(targetNodeKey); @@ -250,16 +254,15 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo } public synchronized void cleanStaleStatistics() { - final DataModificationTransaction trans = dps.beginTransaction(); - final long now = System.nanoTime(); - - flowStats.cleanup(trans, now); - groupDescStats.cleanup(trans, now); - groupStats.cleanup(trans, now); - meterConfigStats.cleanup(trans, now); - meterStats.cleanup(trans, now); - nodeConnectorStats.cleanup(trans, now); - queueStats.cleanup(trans, now); + final DataModificationTransaction trans = this.startDataModification(); + + flowStats.cleanup(trans); + groupDescStats.cleanup(trans); + groupStats.cleanup(trans); + meterConfigStats.cleanup(trans); + meterStats.cleanup(trans); + nodeConnectorStats.cleanup(trans); + queueStats.cleanup(trans); msgManager.cleanStaleTransactionIds(); trans.commit(); @@ -268,26 +271,23 @@ public final class NodeStatisticsHandler implements AutoCloseable, FlowCapableCo public synchronized void requestPeriodicStatistics() { logger.debug("Send requests for statistics collection to node : {}", targetNodeKey); - flowTableStats.request(); - - // FIXME: it does not make sense to trigger this before sendAllFlowTablesStatisticsRequest() - // comes back -- we do not have any tables anyway. - final Collection tables = flowTableStats.getTables(); - logger.debug("Node {} supports {} table(s)", targetNodeKey, tables.size()); - for (final TableKey key : tables) { - logger.debug("Send aggregate stats request for flow table {} to node {}", key.getId(), targetNodeKey); - flowStats.requestAggregateFlows(key); - } - - flowStats.requestAllFlowsAllTables(); - nodeConnectorStats.request(); - groupStats.request(); - groupDescStats.request(); - meterStats.request(); - meterConfigStats.request(); - queueStats.request(); + this.srScheduler.addRequestToSchedulerQueue(flowTableStats); + + this.srScheduler.addRequestToSchedulerQueue(flowStats); + + this.srScheduler.addRequestToSchedulerQueue(nodeConnectorStats); + + this.srScheduler.addRequestToSchedulerQueue(groupStats); + + this.srScheduler.addRequestToSchedulerQueue(groupDescStats); + + this.srScheduler.addRequestToSchedulerQueue(meterStats); + + this.srScheduler.addRequestToSchedulerQueue(meterConfigStats); + + this.srScheduler.addRequestToSchedulerQueue(queueStats); } - + public synchronized void start(final Timer timer) { flowStats.start(dps); groupDescStats.start(dps); diff --git a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/QueueStatsTracker.java b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/QueueStatsTracker.java index f187c7082e..6f93eeb617 100644 --- a/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/QueueStatsTracker.java +++ b/opendaylight/md-sal/statistics-manager/src/main/java/org/opendaylight/controller/md/statistics/manager/QueueStatsTracker.java @@ -36,8 +36,8 @@ final class QueueStatsTracker extends AbstractListeningStatsTracker requestQueue = + Collections.synchronizedMap(new LinkedHashMap()); + + private Long PendingTransactions; + + private long lastRequestTime = System.nanoTime(); + + private static final long REQUEST_MONITOR_INTERVAL = 1000; + + private final TimerTask task = new TimerTask() { + @Override + public void run() { + long now = System.nanoTime(); + if(now > lastRequestTime+TimeUnit.MILLISECONDS.toNanos(REQUEST_MONITOR_INTERVAL)){ + requestStatistics(); + } + } + }; + + public StatisticsRequestScheduler(){ + PendingTransactions = (long) 0; + } + + public void addRequestToSchedulerQueue(AbstractStatsTracker statsRequest){ + requestQueue.put(statsRequest, null); + } + + public AbstractStatsTracker getNextRequestFromSchedulerQueue(){ + //Remove first element + AbstractStatsTracker stats = null; + synchronized(requestQueue){ + Iterator> nodesItr = requestQueue.entrySet().iterator(); + if(nodesItr.hasNext()){ + stats = nodesItr.next().getKey(); + srsLogger.debug("{} chosen up for execution",stats.getNodeRef()); + nodesItr.remove(); + return stats; + } + } + return stats; + } + + private void requestStatistics(){ + AbstractStatsTracker stats = this.getNextRequestFromSchedulerQueue(); + if(stats != null) { + stats.request(); + stats.increaseRequestCounter(); + } + } + @Override + public void onStatusUpdated(DataModificationTransaction transaction, TransactionStatus status) { + + AbstractStatsTracker stats = null; + synchronized(PendingTransactions){ + switch(status){ + case SUBMITED: + this.PendingTransactions++; + break; + case COMMITED: + case FAILED: + this.PendingTransactions--; + if(PendingTransactions == 0){ + lastRequestTime = System.nanoTime(); + stats = this.getNextRequestFromSchedulerQueue(); + } + srsLogger.debug("Pending MD-SAL transactions : {} & Scheduler queue size : {}",this.PendingTransactions,this.requestQueue.size()); + break; + default: + break; + } + } + if(stats != null){ + stats.request(); + stats.increaseRequestCounter(); + } + } + + public void start(){ + timer.schedule(task, 0, REQUEST_MONITOR_INTERVAL); + } +} -- 2.36.6