Upgrade: wait for interface states for adjacencies

This issue was discovered in upgrade CSIT. The issue
is that usually L3 configuration comes in way after
tunnel interface states are present in the operational
database. However, in the case of upgrade this order
can be reversed. The following fix utilizes
DataTreeEventRegistrar to wait for the specific interface
states required to program the Vrfs. Note that this is
only done during upgrade. Also note that this fix has
been tested with the upgrade CSIT and works.

JIRA: NETVIRT-1312

Change-Id: I3b3ba12bbb3453a83dd4969ae6d2487d2fe36dcb
Signed-off-by: Josh <jhershbe@redhat.com>

diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java
index 0625b0ee4b19392b811b39db120d549721fab782..d02272020d1d9e47c74349873d9a52844b67cd48 100644 (file)
--- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java
+++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BaseVrfEntryHandler.java
@@ -15,6 +15,8 @@ import java.math.BigInteger;
 import java.net.Inet4Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -24,6 +26,7 @@ import javax.inject.Singleton;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.mdsalutil.ActionInfo;
@@ -33,6 +36,7 @@ import org.opendaylight.genius.mdsalutil.MDSALUtil;
 import org.opendaylight.genius.mdsalutil.MatchInfo;
 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationEth;
 import org.opendaylight.genius.mdsalutil.actions.ActionMoveSourceDestinationIp;
 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
@@ -59,6 +63,7 @@ import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult;
 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
+import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
@@ -101,17 +106,23 @@ public class BaseVrfEntryHandler implements AutoCloseable {
     private final NexthopManager nextHopManager;
     private final IMdsalApiManager mdsalManager;
     private final FibUtil fibUtil;
+    private final UpgradeState upgradeState;
+    private final DataTreeEventCallbackRegistrar eventCallbacks;
 
     @Inject
     public BaseVrfEntryHandler(final DataBroker dataBroker,
                                final NexthopManager nexthopManager,
                                final IMdsalApiManager mdsalManager,
-                               final FibUtil fibUtil) {
+                               final FibUtil fibUtil,
+                               final UpgradeState upgradeState,
+                               final DataTreeEventCallbackRegistrar eventCallbacks) {
         this.dataBroker = dataBroker;
         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
         this.nextHopManager = nexthopManager;
         this.mdsalManager = mdsalManager;
         this.fibUtil = fibUtil;
+        this.upgradeState = upgradeState;
+        this.eventCallbacks = eventCallbacks;
     }
 
     @Override
@@ -393,10 +404,48 @@ public class BaseVrfEntryHandler implements AutoCloseable {
         addRewriteDstMacAction(vpnId, vrfEntry, prefixInfo, actionInfos);
     }
 
+    private InstanceIdentifier<Interface> getFirstAbsentInterfaceStateIid(List<AdjacencyResult> adjacencyResults) {
+        InstanceIdentifier<Interface> res = null;
+        for (AdjacencyResult adjacencyResult : adjacencyResults) {
+            String interfaceName = adjacencyResult.getInterfaceName();
+            if (null == fibUtil.getInterfaceStateFromOperDS(interfaceName)) {
+                res = fibUtil.buildStateInterfaceId(interfaceName);
+                break;
+            }
+        }
+
+        return res;
+    }
+
     public void programRemoteFib(final BigInteger remoteDpnId, final long vpnId,
                                   final VrfEntry vrfEntry, WriteTransaction tx, String rd,
                                   List<AdjacencyResult> adjacencyResults,
                                   List<SubTransaction> subTxns) {
+        if (upgradeState.isUpgradeInProgress()) {
+            InstanceIdentifier<Interface> absentInterfaceStateIid = getFirstAbsentInterfaceStateIid(adjacencyResults);
+            if (absentInterfaceStateIid != null) {
+                LOG.info("programRemoteFib: interface state for {} not yet present, waiting...",
+                         absentInterfaceStateIid);
+                eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL,
+                    absentInterfaceStateIid,
+                    (before, after) -> {
+                        LOG.info("programRemoteFib: waited for and got interface state {}", absentInterfaceStateIid);
+                        txRunner.callWithNewWriteOnlyTransactionAndSubmit((wtx) -> {
+                            programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null);
+                        });
+                        return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
+                    },
+                    Duration.of(15, ChronoUnit.MINUTES),
+                    (iid) -> {
+                        LOG.error("programRemoteFib: timed out waiting for {}", absentInterfaceStateIid);
+                        txRunner.callWithNewWriteOnlyTransactionAndSubmit((wtx) -> {
+                            programRemoteFib(remoteDpnId, vpnId, vrfEntry, wtx, rd, adjacencyResults, null);
+                        });
+                    });
+                return;
+            }
+        }
+
         List<InstructionInfo> instructions = new ArrayList<>();
         for (AdjacencyResult adjacencyResult : adjacencyResults) {
             List<ActionInfo> actionInfos = new ArrayList<>();

diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BgpRouteVrfEntryHandler.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BgpRouteVrfEntryHandler.java
index 556b7d9a5e94e9268d2dd8bfb75bc12f1dea839d..57d636028c5b81309191d836d20ff6c7dff110e1 100644 (file)
--- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BgpRouteVrfEntryHandler.java
+++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/BgpRouteVrfEntryHandler.java
@@ -25,11 +25,13 @@ import javax.inject.Singleton;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.mdsalutil.ActionInfo;
 import org.opendaylight.genius.mdsalutil.InstructionInfo;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
@@ -79,8 +81,10 @@ public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler
     @Inject
     public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
                                    final NexthopManager nexthopManager,
-                                   final FibUtil fibUtil) {
-        super(dataBroker, nexthopManager, null, fibUtil);
+                                   final FibUtil fibUtil,
+                                   final UpgradeState upgradeState,
+                                   final DataTreeEventCallbackRegistrar eventCallbacks) {
+        super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
         this.dataBroker = dataBroker;
         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
         this.nexthopManager = nexthopManager;

diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/EvpnVrfEntryHandler.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/EvpnVrfEntryHandler.java
index bf51a8cdc762af80d1dbda178c0eca18463369a7..c19378df375d3a166e7e2f44b3ba230c90a6ff03 100644 (file)
--- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/EvpnVrfEntryHandler.java
+++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/EvpnVrfEntryHandler.java
@@ -15,11 +15,13 @@ import java.util.Collections;
 import java.util.List;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.mdsalutil.ActionInfo;
 import org.opendaylight.genius.mdsalutil.InstructionInfo;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
@@ -55,8 +57,9 @@ public class EvpnVrfEntryHandler extends BaseVrfEntryHandler implements IVrfEntr
 
     EvpnVrfEntryHandler(DataBroker broker, VrfEntryListener vrfEntryListener,
             BgpRouteVrfEntryHandler bgpRouteVrfEntryHandler, NexthopManager nexthopManager,
-            JobCoordinator jobCoordinator, IElanService elanManager, FibUtil fibUtil) {
-        super(broker, nexthopManager, null, fibUtil);
+            JobCoordinator jobCoordinator, IElanService elanManager, FibUtil fibUtil,
+            final UpgradeState upgradeState, final DataTreeEventCallbackRegistrar eventCallbacks) {
+        super(broker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
         this.vrfEntryListener = vrfEntryListener;
         this.bgpRouteVrfEntryHandler = bgpRouteVrfEntryHandler;

diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/RouterInterfaceVrfEntryHandler.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/RouterInterfaceVrfEntryHandler.java
index 9da255909cbbae2ea4b51d305ea1b7005bd3fe80..3c25e400be1d89d92cca860fe5f6d102e4f72800 100644 (file)
--- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/RouterInterfaceVrfEntryHandler.java
+++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/RouterInterfaceVrfEntryHandler.java
@@ -15,8 +15,10 @@ import java.util.Collection;
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.mdsalutil.FlowEntity;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
@@ -38,8 +40,9 @@ public class RouterInterfaceVrfEntryHandler extends BaseVrfEntryHandler implemen
 
     @Inject
     public RouterInterfaceVrfEntryHandler(final DataBroker dataBroker, final NexthopManager nexthopManager,
-            final IMdsalApiManager mdsalManager, final IPv6Handler ipv6Handler, final FibUtil fibUtil) {
-        super(dataBroker, nexthopManager, mdsalManager, fibUtil);
+            final IMdsalApiManager mdsalManager, final IPv6Handler ipv6Handler, final FibUtil fibUtil,
+            final UpgradeState upgradeState, final DataTreeEventCallbackRegistrar eventCallbacks) {
+        super(dataBroker, nexthopManager, mdsalManager, fibUtil, upgradeState, eventCallbacks);
         this.mdsalManager = mdsalManager;
         this.ipv6Handler = ipv6Handler;
     }

diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java
index d7e7a0a199dd8afe7b7cfd7608027bac8edd9eed..1196915cd2b32021764c59eb8fe467d3db6ce301 100755 (executable)
--- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java
+++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java
@@ -35,6 +35,7 @@ import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
+import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
 import org.opendaylight.genius.infra.RetryingManagedNewTransactionRunner;
@@ -46,6 +47,7 @@ import org.opendaylight.genius.mdsalutil.MatchInfo;
 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
 import org.opendaylight.genius.mdsalutil.NWUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
+import org.opendaylight.genius.mdsalutil.UpgradeState;
 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
@@ -135,6 +137,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
     private final FibUtil fibUtil;
     private final InterVpnLinkCache interVpnLinkCache;
     private final List<AutoCloseable> closeables = new CopyOnWriteArrayList<>();
+    private final UpgradeState upgradeState;
+    private final DataTreeEventCallbackRegistrar eventCallbacks;
 
     @Inject
     public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager,
@@ -145,7 +149,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
                             final RouterInterfaceVrfEntryHandler routerInterfaceVrfEntryHandler,
                             final JobCoordinator jobCoordinator,
                             final FibUtil fibUtil,
-                            final InterVpnLinkCache interVpnLinkCache) {
+                            final InterVpnLinkCache interVpnLinkCache,
+                            final UpgradeState upgradeState,
+                            final DataTreeEventCallbackRegistrar eventCallbacks) {
         super(VrfEntry.class, VrfEntryListener.class);
         this.dataBroker = dataBroker;
         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
@@ -159,6 +165,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         this.jobCoordinator = jobCoordinator;
         this.fibUtil = fibUtil;
         this.interVpnLinkCache = interVpnLinkCache;
+        this.upgradeState = upgradeState;
+        this.eventCallbacks = eventCallbacks;
     }
 
     @Override
@@ -211,7 +219,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
             LOG.info("EVPN flows need to be programmed.");
             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
-                    nextHopManager, jobCoordinator, elanManager, fibUtil);
+                    nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
             evpnVrfEntryHandler.createFlows(identifier, vrfEntry, rd);
             closeables.add(evpnVrfEntryHandler);
             return;
@@ -245,7 +253,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase<VrfEntry,
         if (VrfEntry.EncapType.Vxlan.equals(vrfEntry.getEncapType())) {
             LOG.info("EVPN flows to be deleted");
             EvpnVrfEntryHandler evpnVrfEntryHandler = new EvpnVrfEntryHandler(dataBroker, this, bgpRouteVrfEntryHandler,
-                    nextHopManager, jobCoordinator, elanManager, fibUtil);
+                    nextHopManager, jobCoordinator, elanManager, fibUtil, upgradeState, eventCallbacks);
             evpnVrfEntryHandler.removeFlows(identifier, vrfEntry, rd);
             closeables.add(evpnVrfEntryHandler);
             return;

diff --git a/fibmanager/impl/src/main/resources/org/opendaylight/blueprint/fibmanager.xml b/fibmanager/impl/src/main/resources/org/opendaylight/blueprint/fibmanager.xml
index 02b745c0bf83404a26800ae13eaa9ec530cf2836..316e2961aa652496a98944acd0e27a1566f11805 100644 (file)
--- a/fibmanager/impl/src/main/resources/org/opendaylight/blueprint/fibmanager.xml
+++ b/fibmanager/impl/src/main/resources/org/opendaylight/blueprint/fibmanager.xml
@@ -20,6 +20,10 @@
              availability="optional"/>
   <reference id="iInterfaceManager"
              interface="org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager" />
+  <reference id="dataTreeEventCallbackRegistrar"
+             interface="org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar"/>
+  <reference id="upgradeState"
+             interface="org.opendaylight.genius.mdsalutil.UpgradeState"/>
 
   <odl:rpc-service id="idManagerService"
                    interface="org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService"/>