BUG-5888: moving the reconciliation process into a different thread to 83/41583/1
authorShuva Kar <shuva.jyoti.kar@ericsson.com>
Tue, 7 Jun 2016 18:28:01 +0000 (23:58 +0530)
committerAnil Vishnoi <vishnoianil@gmail.com>
Fri, 8 Jul 2016 19:47:30 +0000 (12:47 -0700)
prevent holding on to DTCL thread while provisioning bulk flows/groups.

Change-Id: I1ee14e6899fdf9d373634de0260a09ab86303e7f
Signed-off-by: Shuva Kar <shuva.jyoti.kar@ericsson.com>
applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/FlowNodeReconciliation.java
applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowForwarder.java
applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/FlowNodeReconciliationImpl.java
applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/GroupForwarder.java
applications/forwardingrules-manager/src/main/java/org/opendaylight/openflowplugin/applications/frm/impl/MeterForwarder.java

index 9bf1f26732f96af104febb614013420437e016df..b15f2ff21f89ce4751969ac64452e4bfa3bbdc2b 100644 (file)
@@ -8,9 +8,7 @@
 
 package org.opendaylight.openflowplugin.applications.frm;
 
-import org.opendaylight.controller.md.sal.binding.api.ClusteredDataChangeListener;
 import org.opendaylight.controller.md.sal.binding.api.ClusteredDataTreeChangeListener;
-import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 
index 2ca1d120800cb16e4231b48f40428129c342c4c1..f250a1aea88f2ef9d877611d2e7e92d2a56900ab 100644 (file)
@@ -49,8 +49,8 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 
 /**
- * GroupForwarder
- * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataChangeListener}}
+ * FlowForwarder
+ * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener}}
  * for WildCardedPath to {@link Flow} and ForwardingRulesCommiter interface for methods:
  *  add, update and remove {@link Flow} processing for
  *  {@link org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeEvent}.
@@ -84,8 +84,8 @@ public class FlowForwarder extends AbstractListeningCommiter<Flow> {
                 }
             });
         } catch (final Exception e) {
-            LOG.warn("FRM Flow DataChange listener registration fail!");
-            LOG.debug("FRM Flow DataChange listener registration fail ..", e);
+            LOG.warn("FRM Flow DataTreeChange listener registration fail!");
+            LOG.debug("FRM Flow DataTreeChange listener registration fail ..", e);
             throw new IllegalStateException("FlowForwarder startup fail! System needs restart.", e);
         }
     }
index e372ff557dfbc88b11c8da1e802cead2b75b5ec5..dc620b6790a92dbda42c26a2f57bbed0960e8c92 100644 (file)
@@ -9,6 +9,8 @@
 package org.opendaylight.openflowplugin.applications.frm.impl;
 
 import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
@@ -22,8 +24,6 @@ import com.google.common.util.concurrent.Futures;
 import java.util.concurrent.Callable;
 
 import org.opendaylight.controller.md.sal.binding.api.*;
-import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope;
-import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeEvent;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
 import org.opendaylight.openflowplugin.applications.frm.FlowNodeReconciliation;
@@ -33,7 +33,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.acti
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNodeConnector;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.Meter;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.meters.MeterBuilder;
@@ -55,15 +54,12 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group
 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.StaleGroup;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.StaleGroupKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.meter.types.rev130918.MeterId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.table.types.rev131026.table.features.TableFeatures;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.table.types.rev131026.table.features.TableFeaturesKey;
 import org.opendaylight.yangtools.concepts.ListenerRegistration;
-import org.opendaylight.yangtools.yang.binding.DataObject;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.yangtools.yang.binding.KeyedInstanceIdentifier;
 import org.slf4j.Logger;
@@ -94,6 +90,9 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
 
     private ListenerRegistration<FlowNodeReconciliationImpl> listenerRegistration;
 
+    private final int THREAD_POOL_SIZE = 4;
+    ExecutorService executor = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
+
     private static final InstanceIdentifier<FlowCapableNode> II_TO_FLOW_CAPABLE_NODE
             = InstanceIdentifier.builder(Nodes.class)
             .child(Node.class)
@@ -220,40 +219,49 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
                         connectedNode.toString());
                 reconciliationPreProcess(connectedNode);
             }
-            reconciliation(connectedNode);
+            ReconciliationTask reconciliationTask = new ReconciliationTask(connectedNode);
+            executor.execute(reconciliationTask);
         }
     }
 
-    private void reconciliation(final InstanceIdentifier<FlowCapableNode> nodeIdent) {
+    private class ReconciliationTask implements Runnable {
 
-        String sNode = nodeIdent.firstKeyOf(Node.class, NodeKey.class).getId().getValue();
-        long nDpId = getDpnIdFromNodeName(sNode);
+        InstanceIdentifier<FlowCapableNode> nodeIdentity;
 
-        ReadOnlyTransaction trans = provider.getReadTranaction();
-        Optional<FlowCapableNode> flowNode = Optional.absent();
-
-        AtomicInteger counter = new AtomicInteger();
-        //initialize the counter
-        counter.set(0);
-        try {
-            flowNode = trans.read(LogicalDatastoreType.CONFIGURATION, nodeIdent).get();
-        }
-        catch (Exception e) {
-            LOG.error("Fail with read Config/DS for Node {} !", nodeIdent, e);
+        public ReconciliationTask(final InstanceIdentifier<FlowCapableNode> nodeIdent) {
+           nodeIdentity = nodeIdent;
         }
 
-        if (flowNode.isPresent()) {
-            /* Tables - have to be pushed before groups */
-            // CHECK if while pusing the update, updateTableInput can be null to emulate a table add
-            List<TableFeatures> tableList = flowNode.get().getTableFeatures() != null
-                    ? flowNode.get().getTableFeatures() : Collections.<TableFeatures> emptyList() ;
-            for (TableFeatures tableFeaturesItem : tableList) {
-                TableFeaturesKey tableKey = tableFeaturesItem.getKey();
-                KeyedInstanceIdentifier<TableFeatures, TableFeaturesKey> tableFeaturesII
-                    = nodeIdent.child(TableFeatures.class, new TableFeaturesKey(tableKey.getTableId()));
-                        provider.getTableFeaturesCommiter().update(tableFeaturesII, tableFeaturesItem, null, nodeIdent);
+        @Override
+        public void run() {
+
+            String sNode = nodeIdentity.firstKeyOf(Node.class, NodeKey.class).getId().getValue();
+            long nDpId = getDpnIdFromNodeName(sNode);
+
+            ReadOnlyTransaction trans = provider.getReadTranaction();
+            Optional<FlowCapableNode> flowNode = Optional.absent();
+
+            AtomicInteger counter = new AtomicInteger();
+            //initialize the counter
+            counter.set(0);
+            try {
+                flowNode = trans.read(LogicalDatastoreType.CONFIGURATION, nodeIdentity).get();
+            } catch (Exception e) {
+                LOG.error("Fail with read Config/DS for Node {} !", nodeIdentity, e);
             }
 
+            if (flowNode.isPresent()) {
+            /* Tables - have to be pushed before groups */
+                // CHECK if while pusing the update, updateTableInput can be null to emulate a table add
+                List<TableFeatures> tableList = flowNode.get().getTableFeatures() != null
+                        ? flowNode.get().getTableFeatures() : Collections.<TableFeatures>emptyList();
+                for (TableFeatures tableFeaturesItem : tableList) {
+                    TableFeaturesKey tableKey = tableFeaturesItem.getKey();
+                    KeyedInstanceIdentifier<TableFeatures, TableFeaturesKey> tableFeaturesII
+                            = nodeIdentity.child(TableFeatures.class, new TableFeaturesKey(tableKey.getTableId()));
+                    provider.getTableFeaturesCommiter().update(tableFeaturesII, tableFeaturesItem, null, nodeIdentity);
+                }
+
             /* Groups - have to be first */
                 List<Group> groups = flowNode.get().getGroup() != null
                         ? flowNode.get().getGroup() : Collections.<Group>emptyList();
@@ -264,10 +272,10 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
                 List<Group> suspectedGroups = new ArrayList<>();
 
                 while ((!(toBeInstalledGroups.isEmpty()) || !(suspectedGroups.isEmpty())) &&
-                        (counter.get()<=provider.getConfiguration().getReconciliationRetryCount())) { //also check if the counter has not crossed the threshold
+                        (counter.get() <= provider.getConfiguration().getReconciliationRetryCount())) { //also check if the counter has not crossed the threshold
 
-                    if(toBeInstalledGroups.isEmpty() && ! suspectedGroups.isEmpty()){
-                        LOG.error("These Groups are pointing to node-connectors that are not up yet {}",suspectedGroups.toString());
+                    if (toBeInstalledGroups.isEmpty() && !suspectedGroups.isEmpty()) {
+                        LOG.error("These Groups are pointing to node-connectors that are not up yet {}", suspectedGroups.toString());
                         toBeInstalledGroups.addAll(suspectedGroups);
                         break;
                     }
@@ -278,27 +286,27 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
                         boolean okToInstall = true;
                         for (Bucket bucket : group.getBuckets().getBucket()) {
                             for (Action action : bucket.getAction()) {
-                               //chained-port
+                                //chained-port
                                 if (action.getAction().getImplementedInterface().getName()
-                                        .equals("org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase")){
-                                    String nodeConnectorUri = ((OutputActionCase)(action.getAction()))
+                                        .equals("org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase")) {
+                                    String nodeConnectorUri = ((OutputActionCase) (action.getAction()))
                                             .getOutputAction().getOutputNodeConnector().getValue();
 
-                                    LOG.warn("Installing the group for node connector {}",nodeConnectorUri);
+                                    LOG.warn("Installing the group for node connector {}", nodeConnectorUri);
 
                                     //check if the nodeconnector is there in the multimap
                                     boolean isPresent = provider.getFlowNodeConnectorInventoryTranslatorImpl()
                                             .isNodeConnectorUpdated(nDpId, nodeConnectorUri);
                                     //if yes set okToInstall = true
 
-                                    if(isPresent){
-                                       break;
+                                    if (isPresent) {
+                                        break;
                                     }//else put it in a different list and still set okToInstall = true
                                     else {
                                         suspectedGroups.add(group);
                                         LOG.error("Not yet received the node-connector updated for {} " +
-                                                "for the group with id {}",nodeConnectorUri,group.getGroupId().toString());
-                                         break;
+                                                "for the group with id {}", nodeConnectorUri, group.getGroupId().toString());
+                                        break;
                                     }
 
 
@@ -313,21 +321,20 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
                                     }
                                 }
                             }
-                            if (!okToInstall){
+                            if (!okToInstall) {
                                 //increment retry counter value
                                 counter.incrementAndGet();
                                 break;
                             }
 
 
-
                         }
 
 
                         if (okToInstall) {
                             final KeyedInstanceIdentifier<Group, GroupKey> groupIdent =
-                                    nodeIdent.child(Group.class, group.getKey());
-                            this.provider.getGroupCommiter().add(groupIdent, group, nodeIdent);
+                                    nodeIdentity.child(Group.class, group.getKey());
+                            provider.getGroupCommiter().add(groupIdent, group, nodeIdentity);
                             alreadyInstalledGroupids.add(group.getGroupId().getValue());
                             iterator.remove();
                             // resetting the counter to zero
@@ -337,48 +344,48 @@ public class FlowNodeReconciliationImpl implements FlowNodeReconciliation {
                 }
 
             /* installation of suspected groups*/
-            if(!toBeInstalledGroups.isEmpty()){
-                for(Group group :toBeInstalledGroups){
-                    LOG.error("Installing the group {} finally although the port is not up after checking for {} times "
-                            ,group.getGroupId().toString(),provider.getConfiguration().getReconciliationRetryCount());
-                    final KeyedInstanceIdentifier<Group, GroupKey> groupIdent =
-                            nodeIdent.child(Group.class, group.getKey());
-                    this.provider.getGroupCommiter().add(groupIdent, group, nodeIdent);
+                if (!toBeInstalledGroups.isEmpty()) {
+                    for (Group group : toBeInstalledGroups) {
+                        LOG.error("Installing the group {} finally although the port is not up after checking for {} times "
+                                , group.getGroupId().toString(), provider.getConfiguration().getReconciliationRetryCount());
+                        final KeyedInstanceIdentifier<Group, GroupKey> groupIdent =
+                                nodeIdentity.child(Group.class, group.getKey());
+                        provider.getGroupCommiter().add(groupIdent, group, nodeIdentity);
+                    }
                 }
-            }
             /* Meters */
-            List<Meter> meters = flowNode.get().getMeter() != null
-                    ? flowNode.get().getMeter() : Collections.<Meter> emptyList();
-            for (Meter meter : meters) {
-                final KeyedInstanceIdentifier<Meter, MeterKey> meterIdent =
-                        nodeIdent.child(Meter.class, meter.getKey());
-                this.provider.getMeterCommiter().add(meterIdent, meter, nodeIdent);
-            }
+                List<Meter> meters = flowNode.get().getMeter() != null
+                        ? flowNode.get().getMeter() : Collections.<Meter>emptyList();
+                for (Meter meter : meters) {
+                    final KeyedInstanceIdentifier<Meter, MeterKey> meterIdent =
+                            nodeIdentity.child(Meter.class, meter.getKey());
+                    provider.getMeterCommiter().add(meterIdent, meter, nodeIdentity);
+                }
             /* Flows */
-            List<Table> tables = flowNode.get().getTable() != null
-                    ? flowNode.get().getTable() : Collections.<Table> emptyList();
-            for (Table table : tables) {
-                final KeyedInstanceIdentifier<Table, TableKey> tableIdent =
-                        nodeIdent.child(Table.class, table.getKey());
-                List<Flow> flows = table.getFlow() != null ? table.getFlow() : Collections.<Flow> emptyList();
-                for (Flow flow : flows) {
-                    final KeyedInstanceIdentifier<Flow, FlowKey> flowIdent =
-                            tableIdent.child(Flow.class, flow.getKey());
-                    this.provider.getFlowCommiter().add(flowIdent, flow, nodeIdent);
+                List<Table> tables = flowNode.get().getTable() != null
+                        ? flowNode.get().getTable() : Collections.<Table>emptyList();
+                for (Table table : tables) {
+                    final KeyedInstanceIdentifier<Table, TableKey> tableIdent =
+                            nodeIdentity.child(Table.class, table.getKey());
+                    List<Flow> flows = table.getFlow() != null ? table.getFlow() : Collections.<Flow>emptyList();
+                    for (Flow flow : flows) {
+                        final KeyedInstanceIdentifier<Flow, FlowKey> flowIdent =
+                                tableIdent.child(Flow.class, flow.getKey());
+                        provider.getFlowCommiter().add(flowIdent, flow, nodeIdentity);
+                    }
                 }
             }
-        }
         /* clean transaction */
-        trans.close();
+            trans.close();
+        }
     }
-       private long getDpnIdFromNodeName(String nodeName) {
+    private long getDpnIdFromNodeName(String nodeName) {
         String dpId = nodeName.substring(nodeName.lastIndexOf(SEPARATOR) + 1);
-               return Long.parseLong(dpId);
-       }
+        return Long.parseLong(dpId);
+    }
 
     private void reconciliationPreProcess(final InstanceIdentifier<FlowCapableNode> nodeIdent) {
 
-
         List<InstanceIdentifier<StaleFlow>> staleFlowsToBeBulkDeleted = Lists.newArrayList();
         List<InstanceIdentifier<StaleGroup>> staleGroupsToBeBulkDeleted = Lists.newArrayList();
         List<InstanceIdentifier<StaleMeter>> staleMetersToBeBulkDeleted = Lists.newArrayList();
index 0441d84da6b8452ffcdf8d5c7ca7c6e7b45142cd..96bb0628c239b927b5ae8a2391162f1bba27cd19 100644 (file)
@@ -46,7 +46,7 @@ import java.util.concurrent.Future;
 
 /**
  * GroupForwarder
- * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataChangeListener}}
+ * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener}}
  * for WildCardedPath to {@link Group} and ForwardingRulesCommiter interface for methods:
  *  add, update and remove {@link Group} processing for
  *  {@link org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeEvent}.
@@ -77,9 +77,9 @@ public class GroupForwarder extends AbstractListeningCommiter<Group> {
                 }
             });
         } catch (final Exception e) {
-            LOG.warn("FRM Group DataChange listener registration fail!");
-            LOG.debug("FRM Group DataChange listener registration fail ..", e);
-            throw new IllegalStateException("FlowForwarder startup fail! System needs restart.", e);
+            LOG.warn("FRM Group DataTreeChange listener registration fail!");
+            LOG.debug("FRM Group DataTreeChange listener registration fail ..", e);
+            throw new IllegalStateException("GroupForwarder startup fail! System needs restart.", e);
         }
     }
 
index d571f0b752d6a8d33f5a82f3282fae80a581c7cf..de467ba9a9afe1c55bee63964ae07b74908c214b 100644 (file)
@@ -46,7 +46,7 @@ import java.util.concurrent.Future;
 
 /**
  * MeterForwarder
- * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataChangeListener}}
+ * It implements {@link org.opendaylight.controller.md.sal.binding.api.DataTreeChangeListener}}
  * for WildCardedPath to {@link Meter} and ForwardingRulesCommiter interface for methods:
  *  add, update and remove {@link Meter} processing for
  *  {@link org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeEvent}.
@@ -77,9 +77,9 @@ public class MeterForwarder extends AbstractListeningCommiter<Meter> {
                 }
             });
         } catch (final Exception e) {
-            LOG.warn("FRM Meter DataChange listener registration fail!");
-            LOG.debug("FRM Meter DataChange listener registration fail ..", e);
-            throw new IllegalStateException("FlowForwarder startup fail! System needs restart.", e);
+            LOG.warn("FRM Meter DataTreeChange listener registration fail!");
+            LOG.debug("FRM Meter DataTreeChange listener registration fail ..", e);
+            throw new IllegalStateException("MeterForwarder startup fail! System needs restart.", e);
         }
     }
 
@@ -157,7 +157,7 @@ public class MeterForwarder extends AbstractListeningCommiter<Meter> {
 
     @Override
     public void createStaleMarkEntity(InstanceIdentifier<Meter> identifier, Meter del, InstanceIdentifier<FlowCapableNode> nodeIdent) {
-        LOG.debug("Creating Stale-Mark entry for the switch {} for flow {} ", nodeIdent.toString(), del.toString());
+        LOG.debug("Creating Stale-Mark entry for the switch {} for meter {} ", nodeIdent.toString(), del.toString());
         StaleMeter staleMeter = makeStaleMeter(identifier, del, nodeIdent);
         persistStaleMeter(staleMeter, nodeIdent);
     }