Upgrade: race cond. btwn elan-dpn-interfaces and arp-responder for external ifaces

Depends on this gerrit from genius:
https://git.opendaylight.org/gerrit/#/c/65894/

A comment from inside the code:
The following through the end of the function deals with
an upgrade scenario where the neutron configuration
is restored before the OVS switches reconnect. In such a
case, the elan-dpn-interfaces entries will be
missing from the operational data store. In order to
mitigate this we use DataTreeEventCallbackRegistrar
to wait for the exact operational md-sal object we need
to contain the external interface we need.

Change-Id: Id02564a13e7f85933867e21c51e36c32bc60cd4e
Signed-off-by: Josh <jhershbe@redhat.com>

diff --git a/vpnservice/elanmanager/elanmanager-api/src/main/java/org/opendaylight/netvirt/elanmanager/api/IElanService.java b/vpnservice/elanmanager/elanmanager-api/src/main/java/org/opendaylight/netvirt/elanmanager/api/IElanService.java
index 0266fc5b89be2be2c68157cf48c5123407e6d4c9..61d1ff1bb1fd03269bc4b846b0c275ddc66f7954 100644 (file)
--- a/vpnservice/elanmanager/elanmanager-api/src/main/java/org/opendaylight/netvirt/elanmanager/api/IElanService.java
+++ b/vpnservice/elanmanager/elanmanager-api/src/main/java/org/opendaylight/netvirt/elanmanager/api/IElanService.java
@@ -15,10 +15,12 @@ import java.util.List;
 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
 import org.opendaylight.netvirt.elanmanager.exceptions.MacNotFoundException;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 
 public interface IElanService extends IEtreeService {
 
@@ -102,4 +104,7 @@ public interface IElanService extends IEtreeService {
 
     Long retrieveNewElanTag(String idKey);
 
+    InstanceIdentifier<DpnInterfaces> getElanDpnInterfaceOperationalDataPath(String elanInstanceName, BigInteger dpnId);
+
+    DpnInterfaces getElanInterfaceInfoByElanDpn(String elanInstanceName, BigInteger dpId);
 }

diff --git a/vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanServiceProvider.java b/vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanServiceProvider.java
index f904aa90637631937fb8b34ac155508823efc6e8..e49af511d6479ea34a59ee364ea8732b355e9acd 100644 (file)
--- a/vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanServiceProvider.java
+++ b/vpnservice/elanmanager/elanmanager-impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanServiceProvider.java
@@ -882,4 +882,15 @@ public class ElanServiceProvider extends AbstractLifecycle implements IElanServi
         return elanUtils.retrieveNewElanTag(idManager, idKey);
     }
 
+    @Override
+    public InstanceIdentifier<DpnInterfaces> getElanDpnInterfaceOperationalDataPath(
+                                                                String elanInstanceName, BigInteger dpnId) {
+        return ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpnId);
+    }
+
+    @Override
+    public DpnInterfaces getElanInterfaceInfoByElanDpn(String elanInstanceName, BigInteger dpId) {
+        return elanUtils.getElanInterfaceInfoByElanDpn(elanInstanceName, dpId);
+    }
+
 }

diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java
index ce862deb148523a5879d18c3792614cac95218ec..8a2edb4f298e3380de47a31f368b25d450c1add1 100644 (file)
--- a/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java
+++ b/vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java
@@ -22,6 +22,7 @@ import javax.inject.Singleton;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
@@ -30,6 +31,7 @@ import org.opendaylight.genius.mdsalutil.MDSALUtil;
 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.genius.mdsalutil.nxmatches.NxMatchRegister;
@@ -58,6 +60,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
@@ -87,6 +90,8 @@ public class VpnManagerImpl implements IVpnManager {
     private final IFibManager fibManager;
     private final IBgpManager bgpManager;
     private final InterVpnLinkCache interVpnLinkCache;
+    private final DataTreeEventCallbackRegistrar eventCallbacks;
+    private final UpgradeState upgradeState;
 
     @Inject
     public VpnManagerImpl(final DataBroker dataBroker,
@@ -100,7 +105,9 @@ public class VpnManagerImpl implements IVpnManager {
                           final IVpnLinkService ivpnLinkService,
                           final IFibManager fibManager,
                           final IBgpManager bgpManager,
-                          final InterVpnLinkCache interVpnLinkCache) {
+                          final InterVpnLinkCache interVpnLinkCache,
+                          final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
+                          final UpgradeState upgradeState) {
         this.dataBroker = dataBroker;
         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
         this.idManager = idManagerService;
@@ -113,6 +120,8 @@ public class VpnManagerImpl implements IVpnManager {
         this.fibManager = fibManager;
         this.bgpManager = bgpManager;
         this.interVpnLinkCache = interVpnLinkCache;
+        this.eventCallbacks = dataTreeEventCallbackRegistrar;
+        this.upgradeState = upgradeState;
     }
 
     @PostConstruct
@@ -473,39 +482,71 @@ public class VpnManagerImpl implements IVpnManager {
         }
 
         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
-        if (extInterfaceName == null) {
-            LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
+        if (extInterfaceName != null) {
+            doAddArpResponderFlowsToExternalNetworkIps(
+                    id, fixedIps, macAddress, dpnId, extNetworkId, writeTx, extInterfaceName);
             return;
         }
 
-        Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
-        if (extInterfaceState == null) {
-            LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
-                    id);
-            return;
-        }
+        LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 
-        Integer lportTag = extInterfaceState.getIfIndex();
-        if (lportTag == null) {
-            LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
+        if (!upgradeState.isUpgradeInProgress()) {
             return;
         }
 
-        if (macAddress == null) {
+        // The following through the end of the function deals with an upgrade scenario where the neutron configuration
+        // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
+        // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
+        // to wait for the exact operational md-sal object we need to contain the external interface we need.
 
-            LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
-                    + "External Gw MacAddress is missing.", fixedIps,  id);
-            return;
-        }
+        LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
+                dpnId, extNetworkId.getValue());
+
+        InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
+                            elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
+
+        eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
+            LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
+                    extNetworkId);
+            DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
+            if (dpnInterfaces == null) {
+                LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
+                return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
+            }
+
+            String extIfc = null;
+            for (String dpnInterface : dpnInterfaces.getInterfaces()) {
+                if (interfaceManager.isExternalInterface(dpnInterface)) {
+                    extIfc = dpnInterface;
+                    break;
+                }
+            }
+
+            if (extIfc == null) {
+                if (upgradeState.isUpgradeInProgress()) {
+                    LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
+                            extNetworkId.getValue(), dpnInterfaces);
+                    return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
+                } else {
+                    return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
+                }
+            }
+
+            final String extIfcFinal = extIfc;
+            ListenableFuture<Void> listenableFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
+                doAddArpResponderFlowsToExternalNetworkIps(
+                        id, fixedIps, macAddress, dpnId, extNetworkId, tx, extIfcFinal);
+            });
+            ListenableFutures.addErrorLogging(listenableFuture, LOG,
+                    "Error while configuring arp responder for ext. interface");
+
+            return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER; });
 
-        long vpnId = getVpnIdFromExtNetworkId(extNetworkId);
-        addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, vpnId, extInterfaceName, lportTag,
-                writeTx);
     }
 
     @Override
     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
-            BigInteger dpnId, long vpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
+                     BigInteger dpnId, long vpnId, String extInterfaceName, int lportTag, WriteTransaction writeTx) {
         if (fixedIps == null || fixedIps.isEmpty()) {
             LOG.debug("No external IPs defined for {}", id);
             return;
@@ -530,6 +571,33 @@ public class VpnManagerImpl implements IVpnManager {
         }
     }
 
+    private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
+                            BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx, String extInterfaceName) {
+        Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
+        if (extInterfaceState == null) {
+            LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
+                    id);
+            return;
+        }
+
+        Integer lportTag = extInterfaceState.getIfIndex();
+        if (lportTag == null) {
+            LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
+            return;
+        }
+
+        if (macAddress == null) {
+
+            LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
+                    + "External Gw MacAddress is missing.", fixedIps,  id);
+            return;
+        }
+
+        long vpnId = getVpnIdFromExtNetworkId(extNetworkId);
+        addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, vpnId, extInterfaceName, lportTag,
+                writeTx);
+    }
+
     @Override
     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
             BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {

diff --git a/vpnservice/vpnmanager/vpnmanager-impl/src/main/resources/org/opendaylight/blueprint/vpnmanager.xml b/vpnservice/vpnmanager/vpnmanager-impl/src/main/resources/org/opendaylight/blueprint/vpnmanager.xml
index 5b5d9d92c366be452e0faa01ba05bf86f3b4955e..d6a092bae3868ab062beb459bd9bf7ab51dc67e7 100644 (file)
--- a/vpnservice/vpnmanager/vpnmanager-impl/src/main/resources/org/opendaylight/blueprint/vpnmanager.xml
+++ b/vpnservice/vpnmanager/vpnmanager-impl/src/main/resources/org/opendaylight/blueprint/vpnmanager.xml
@@ -25,6 +25,10 @@
              interface="org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService" />
   <reference id="jobCoordinator"
              interface="org.opendaylight.infrautils.jobcoordinator.JobCoordinator"/>
+  <reference id="dataTreeEventCallbackRegistrar"
+             interface="org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar"/>
+  <reference id="upgradeState"
+             interface="org.opendaylight.genius.mdsalutil.UpgradeState"/>
 
   <odl:rpc-service id="idManagerService"
                    interface="org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService" />