RPC Registration changes for clustering

1. Unregisters rpcs for master/equal to slave scenario as well.
2. Made RpcManager as the device-context-closed-handler , instead of the RpcContext
3. If Role Change fails after several attempts on a switch, the device context is closed. This is to avoid rpcs getting registered in wrong nodes.
4. Removed debug exception for device close in RoleManagerImpl
5. Addition of Logs

Change-Id: Ie5af7466b1422698d3c3287195d237fe1407b023
Signed-off-by: Kamal Rameshan <kramesha@cisco.com>

diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcContext.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcContext.java
index ec8c7e7bcf65713755fef35d998f48cda21c688a..30458e01e6009bd121f0eee7fbd4ec3667d369d4 100644 (file)
--- a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcContext.java
+++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcContext.java
@@ -18,6 +18,6 @@ import org.opendaylight.yangtools.yang.binding.RpcService;
  * <p>
  * Created by Martin Bobak &lt;mbobak@cisco.com&gt; on 25.2.2015.
  */
-public interface RpcContext extends RequestContextStack, AutoCloseable, DeviceContextClosedHandler {
+public interface RpcContext extends RequestContextStack, AutoCloseable {
     <S extends RpcService> void registerRpcServiceImplementation(Class<S> serviceClass, S serviceInstance);
 }

diff --git a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcManager.java b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcManager.java
index b4d0d5ad28e4825e98c048386088ff37a66b0519..928fd716d8b4b69955fa450c86ac61b45bcc3344 100644 (file)
--- a/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcManager.java
+++ b/openflowplugin-api/src/main/java/org/opendaylight/openflowplugin/api/openflow/rpc/RpcManager.java
@@ -8,6 +8,7 @@
 
 package org.opendaylight.openflowplugin.api.openflow.rpc;
 
+import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceContextClosedHandler;
 import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitializationPhaseHandler;
 import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitializator;
 
@@ -18,6 +19,6 @@ import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitia
  * <p>
  * Created by Martin Bobak &lt;mbobak@cisco.com&gt; on 25.2.2015.
  */
-public interface RpcManager extends DeviceInitializator, DeviceInitializationPhaseHandler, AutoCloseable {
+public interface RpcManager extends DeviceInitializator, DeviceInitializationPhaseHandler, AutoCloseable, DeviceContextClosedHandler {
 
 }

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleContextImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleContextImpl.java
index 89209338da4be49814bba8ae2612d4313ead65b1..fd392cd19f3aaa5ea29c735e1a59173b16cd2625 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleContextImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleContextImpl.java
@@ -130,7 +130,6 @@ public class RoleContextImpl implements RoleContext {
     public void close() throws Exception {
         if (entityOwnershipCandidateRegistration != null) {
             LOG.debug("Closing EntityOwnershipCandidateRegistration for {}", entity);
-            LOG.error("Who called this close?????", new Throwable());
             entityOwnershipCandidateRegistration.close();
         }
     }

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleManagerImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleManagerImpl.java
index 30929433c759b4de51acb75e37e33e7e42947185..dfdf345f30dcd9cd3d292621bdfab231bd9c50d9 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleManagerImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/role/RoleManagerImpl.java
@@ -76,12 +76,12 @@ public class RoleManagerImpl implements RoleManager {
                 LOG.error("RoleChange on device {} was not successful after several attempts. " +
                         "Closing the device Context, reconnect the device and start over",
                         deviceContext.getPrimaryConnectionContext().getNodeId().getValue(), throwable);
-//                try {
-//                    deviceContext.close();
-//                } catch (Exception e) {
-//                    LOG.warn("Error closing device context for device:{}",
-//                            deviceContext.getPrimaryConnectionContext().getNodeId().getValue(),  e);
-//                }
+                try {
+                    deviceContext.close();
+                } catch (Exception e) {
+                    LOG.warn("Error closing device context for device:{}",
+                            deviceContext.getPrimaryConnectionContext().getNodeId().getValue(),  e);
+                }
             }
         });
     }

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java
index 502ac137241296d82d4b68fa8ad3db3464b0aece..b373bf93db35c3b868cef95d7faf3a802e8fa82d 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcContextImpl.java
@@ -68,6 +68,8 @@ public class RpcContextImpl implements RpcContext {
         for (final RoutedRpcRegistration<?> rpcRegistration : rpcRegistrations) {
             rpcRegistration.unregisterPath(NodeContext.class, deviceContext.getDeviceState().getNodeInstanceIdentifier());
             rpcRegistration.close();
+            LOG.debug("Closing RPC Registration of service {} for device {}.", rpcRegistration.getServiceType(),
+                    deviceContext.getDeviceState().getNodeInstanceIdentifier());
         }
     }
 
@@ -87,9 +89,4 @@ public class RpcContextImpl implements RpcContext {
             }
         };
     }
-
-    @Override
-    public void onDeviceContextClosed(DeviceContext deviceContext) {
-        close();
-    }
 }

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImpl.java
index dbc1c5421c0dac41635270c6b288dc9123863783..3031214d09f4b18503a50ae1030df7f418cab870 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/rpc/RpcManagerImpl.java
@@ -12,17 +12,22 @@ import org.opendaylight.openflowplugin.api.openflow.device.DeviceContext;
 import org.opendaylight.openflowplugin.api.openflow.device.handlers.DeviceInitializationPhaseHandler;
 import org.opendaylight.openflowplugin.api.openflow.rpc.RpcContext;
 import org.opendaylight.openflowplugin.api.openflow.rpc.RpcManager;
+import org.opendaylight.openflowplugin.api.openflow.statistics.StatisticsContext;
 import org.opendaylight.openflowplugin.impl.util.MdSalRegistratorUtils;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.role.service.rev150727.OfpRole;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.concurrent.ConcurrentHashMap;
+
 public class RpcManagerImpl implements RpcManager {
 
     private static final Logger LOG = LoggerFactory.getLogger(RpcManagerImpl.class);
     private final RpcProviderRegistry rpcProviderRegistry;
     private DeviceInitializationPhaseHandler deviceInitPhaseHandler;
     private final int maxRequestsQuota;
+    private final ConcurrentHashMap<DeviceContext, RpcContext> contexts = new ConcurrentHashMap<>();
 
     public RpcManagerImpl(final RpcProviderRegistry rpcProviderRegistry,
                           final int quotaValue) {
@@ -37,16 +42,34 @@ public class RpcManagerImpl implements RpcManager {
 
     @Override
     public void onDeviceContextLevelUp(final DeviceContext deviceContext) {
-        LOG.debug("deviceContext.getDeviceState().getRole():"+deviceContext.getDeviceState().getRole());
-        if (deviceContext.getDeviceState().getRole() == OfpRole.BECOMESLAVE) {
-            // if slave, we dont poll for statistics and jump to rpc initialization
-            deviceInitPhaseHandler.onDeviceContextLevelUp(deviceContext);
-            return;
+        NodeId nodeId = deviceContext.getDeviceState().getNodeId();
+        OfpRole ofpRole = deviceContext.getDeviceState().getRole();
+
+        LOG.debug("Node:{}, deviceContext.getDeviceState().getRole():{}", nodeId, ofpRole);
+
+        RpcContext rpcContext = contexts.get(deviceContext);
+        if (rpcContext == null) {
+            rpcContext = new RpcContextImpl(deviceContext.getMessageSpy(), rpcProviderRegistry, deviceContext, maxRequestsQuota);
+            contexts.put(deviceContext, rpcContext);
+        }
+
+
+        if (ofpRole == OfpRole.BECOMESLAVE) {
+            // if slave, we need to de-register rpcs if any have been registered, in case of master to slave
+            LOG.info("Unregistering RPC registration (if any) for slave role for node:{}", deviceContext.getDeviceState().getNodeId());
+            try {
+                MdSalRegistratorUtils.unregisterServices(rpcContext);
+            } catch (Exception e) {
+                LOG.error("Exception while unregistering rpcs for slave role for node:{}. But continuing.", nodeId, e);
+            }
+
+        } else {
+            LOG.info("Registering Openflow RPCs for node:{}, role:{}", nodeId, ofpRole);
+            MdSalRegistratorUtils.registerServices(rpcContext, deviceContext);
         }
 
-        final RpcContext rpcContext = new RpcContextImpl(deviceContext.getMessageSpy(), rpcProviderRegistry, deviceContext, maxRequestsQuota);
-        deviceContext.addDeviceContextClosedHandler(rpcContext);
-        MdSalRegistratorUtils.registerServices(rpcContext, deviceContext);
+        deviceContext.addDeviceContextClosedHandler(this);
+
         // finish device initialization cycle back to DeviceManager
         deviceInitPhaseHandler.onDeviceContextLevelUp(deviceContext);
     }
@@ -55,4 +78,20 @@ public class RpcManagerImpl implements RpcManager {
     public void close() throws Exception {
 
     }
+
+
+    @Override
+    public void onDeviceContextClosed(DeviceContext deviceContext) {
+        RpcContext removedContext = contexts.remove(deviceContext);
+        if (removedContext != null) {
+            try {
+                LOG.info("Unregistering rpcs for device context closure");
+                removedContext.close();
+            } catch (Exception e) {
+                LOG.error("Exception while unregistering rpcs onDeviceContextClosed handler for node:{}. But continuing.",
+                        deviceContext.getDeviceState().getNodeId(), e);
+            }
+        }
+
+    }
 }

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java
index 5c98f2b17de35523ef9d47f962876582b93efcda..cfb430d8f71cc073ff457d3b67eb9e8029339199 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/statistics/StatisticsManagerImpl.java
@@ -81,9 +81,11 @@ public class StatisticsManagerImpl implements StatisticsManager, StatisticsManag
 
     @Override
     public void onDeviceContextLevelUp(final DeviceContext deviceContext) {
-        LOG.debug("deviceContext.getDeviceState().getRole():"+deviceContext.getDeviceState().getRole());
+        LOG.debug("Node:{}, deviceContext.getDeviceState().getRole():{}", deviceContext.getDeviceState().getNodeId(),
+                deviceContext.getDeviceState().getRole());
         if (deviceContext.getDeviceState().getRole() == OfpRole.BECOMESLAVE) {
             // if slave, we dont poll for statistics and jump to rpc initialization
+            LOG.info("Skipping Statistics for slave role for node:{}", deviceContext.getDeviceState().getNodeId());
             deviceInitPhaseHandler.onDeviceContextLevelUp(deviceContext);
             return;
         }
@@ -93,6 +95,8 @@ public class StatisticsManagerImpl implements StatisticsManager, StatisticsManag
             hashedWheelTimer = deviceContext.getTimer();
         }
 
+        LOG.info("Starting Statistics for master role for node:{}", deviceContext.getDeviceState().getNodeId());
+
         final StatisticsContext statisticsContext = new StatisticsContextImpl(deviceContext);
         deviceContext.addDeviceContextClosedHandler(this);
         final ListenableFuture<Boolean> weHaveDynamicData = statisticsContext.gatherDynamicData();

diff --git a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/util/MdSalRegistratorUtils.java b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/util/MdSalRegistratorUtils.java
index 4d3e3e8bf3bd3ccd9b506ad064ab080dca5db1ad..7373c44dab2a9f164128a49f3f5ea128d9fbb55f 100644 (file)
--- a/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/util/MdSalRegistratorUtils.java
+++ b/openflowplugin-impl/src/main/java/org/opendaylight/openflowplugin/impl/util/MdSalRegistratorUtils.java
@@ -47,4 +47,8 @@ public class MdSalRegistratorUtils {
         rpcContext.registerRpcServiceImplementation(NodeConfigService.class, new NodeConfigServiceImpl(rpcContext, deviceContext));
         rpcContext.registerRpcServiceImplementation(OpendaylightFlowStatisticsService.class, new OpendaylightFlowStatisticsServiceImpl(rpcContext, deviceContext));
     }
+
+    public static void unregisterServices(final RpcContext rpcContext) throws Exception {
+        rpcContext.close();
+    }
 }