Fix: Tunnels are in UNKNOWN state after CSS upgrade

Tunnels went into unknown state because the connects events were missed
by ITM due to EoS switchover. This fix will recover the unknown tunnels
based on inventory oper data store.

Change-Id: Ifa70f91825f73a2dc8c384d97d2e0b54d5f97074
Signed-off-by: Apurba Mukherjee <apurba.mukherjee@ericsson.com>

diff --git a/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/impl/ItmProvider.java b/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/impl/ItmProvider.java
index d6761a2cf605e3f293fa732998c310d6a625e665..5cab70464108169ffc435c42685efd6dafa3ff58 100644 (file)
--- a/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/impl/ItmProvider.java
+++ b/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/impl/ItmProvider.java
@@ -34,6 +34,7 @@ import org.opendaylight.genius.itm.listeners.TransportZoneListener;
 import org.opendaylight.genius.itm.listeners.TunnelMonitorChangeListener;
 import org.opendaylight.genius.itm.listeners.TunnelMonitorIntervalListener;
 import org.opendaylight.genius.itm.monitoring.ItmTunnelEventListener;
+import org.opendaylight.genius.itm.recovery.impl.EosChangeEventHandler;
 import org.opendaylight.genius.itm.rpc.ItmManagerRpcService;
 import org.opendaylight.infrautils.diagstatus.ServiceState;
 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
@@ -96,7 +97,7 @@ public class ItmProvider implements AutoCloseable, IITMProvider /*,ItmStateServi
     private final ItmProvider.ItmProviderEOSListener itmProviderEOSListener;
     public  Integer batchSize;
     public  Integer batchInterval;
-
+    private EosChangeEventHandler eosChangeEventHandler;
 
     @Inject
     public ItmProvider(DataBroker dataBroker,
@@ -115,7 +116,8 @@ public class ItmProvider implements AutoCloseable, IITMProvider /*,ItmStateServi
                        final ItmDiagStatusProvider itmDiagStatusProvider,
                        final TunnelStateCache tunnelStateCache,
                        final ItmConfig itmConfig,
-                       final JobCoordinator jobCoordinator) {
+                       final JobCoordinator jobCoordinator,
+                       final EosChangeEventHandler eosChangeEventHandler) {
         LOG.info("ItmProvider Before register MBean");
         this.dataBroker = dataBroker;
         this.idManager = idManagerService;
@@ -132,6 +134,7 @@ public class ItmProvider implements AutoCloseable, IITMProvider /*,ItmStateServi
         this.dpnTepStateCache = dpnTepStateCache;
         this.itmStatusProvider = itmDiagStatusProvider;
         this.tunnelStateCache = tunnelStateCache;
+        this.eosChangeEventHandler = eosChangeEventHandler;
 
         this.itmConfig = itmConfig;
         this.jobCoordinator = jobCoordinator;
@@ -373,6 +376,7 @@ public class ItmProvider implements AutoCloseable, IITMProvider /*,ItmStateServi
         if (ownershipChange.getState().isOwner()) {
             LOG.info("*This* instance of provider is set as a MASTER instance");
             createDefaultTransportZone(itmConfig);
+            eosChangeEventHandler.recoverUnknownTunnelsOnEosSwitch();
         } else {
             LOG.info("*This* instance of provider is set as a SLAVE instance");
         }

diff --git a/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/recovery/impl/EosChangeEventHandler.java b/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/recovery/impl/EosChangeEventHandler.java
new file mode 100644 (file)
index 0000000..c2c2929
--- /dev/null
+++ b/itm/itm-impl/src/main/java/org/opendaylight/genius/itm/recovery/impl/EosChangeEventHandler.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2020 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.genius.itm.recovery.impl;
+
+import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.opendaylight.genius.infra.Datastore.Operational;
+import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
+import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
+import org.opendaylight.genius.infra.TypedWriteTransaction;
+import org.opendaylight.genius.itm.cache.TunnelStateCache;
+import org.opendaylight.genius.itm.impl.ItmUtils;
+import org.opendaylight.genius.itm.itmdirecttunnels.renderer.ovs.utilities.DirectTunnelUtils;
+import org.opendaylight.mdsal.binding.api.DataBroker;
+import org.opendaylight.mdsal.binding.api.ReadTransaction;
+import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeConnectorId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnector;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.node.NodeConnectorKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Singleton
+public class EosChangeEventHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(EosChangeEventHandler.class);
+    private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("GeniusEventLogger");
+
+    private final DataBroker dataBroker;
+    private final TunnelStateCache tunnelStateCache;
+    private final ManagedNewTransactionRunner txRunner;
+    private final DirectTunnelUtils directTunnelUtils;
+
+    @Inject
+    public EosChangeEventHandler(DataBroker dataBroker, TunnelStateCache tunnelStateCache,
+                                 DirectTunnelUtils directTunnelUtils) {
+        LOG.info("registering EOS change handlers");
+        this.tunnelStateCache = tunnelStateCache;
+        this.dataBroker = dataBroker;
+        this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
+        this.directTunnelUtils = directTunnelUtils;
+    }
+
+    public void recoverUnknownTunnelsOnEosSwitch() {
+        LOG.debug("EosChangeListener: recovering unknown tunnels");
+
+        if (!directTunnelUtils.isEntityOwner()) {
+            LOG.debug("Not an entity owner, returning...");
+            return;
+        }
+        List<StateTunnelList> unknownTunnels = getUnknownTunnelsFromDefaultOper();
+        if (unknownTunnels.isEmpty()) {
+            LOG.debug("Empty oper DS or No unknown tunnels in ITM oper DS");
+        } else {
+            ConcurrentMap<String, String> tunnelNameToNodeConnectorIdValue = prepareKeyForInventoryOper(unknownTunnels);
+            Set<ConcurrentMap.Entry<String,String>> entrySet = tunnelNameToNodeConnectorIdValue.entrySet();
+            ReadTransaction readOnlyTx = dataBroker.newReadOnlyTransaction();
+            for (ConcurrentMap.Entry<String,String> entry : entrySet) {
+                String tunnelName = entry.getKey();
+                String nodeConnectorIdValue = entry.getValue();
+                String nodeIdValue = nodeConnectorIdValue.substring(0, nodeConnectorIdValue.lastIndexOf(":"));
+                NodeConnectorId nodeConnectorId = new NodeConnectorId(nodeConnectorIdValue);
+                NodeId nodeId = new NodeId(nodeIdValue);
+                InstanceIdentifier<NodeConnector> ncIdentifier = InstanceIdentifier.builder(Nodes.class)
+                        .child(Node.class, new NodeKey(nodeId))
+                        .child(NodeConnector.class, new NodeConnectorKey(nodeConnectorId)).build();
+
+                try {
+                    if (readOnlyTx.exists(LogicalDatastoreType.OPERATIONAL, ncIdentifier).get()) {
+                        txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, tx -> {
+                            updateTunnelState(tunnelName, tx);
+                        });
+                    }
+                } catch (InterruptedException e) {
+                    LOG.error("EosChangeListener: Inventory oper read failed for {}, Reason: {}",
+                            ncIdentifier, e.getMessage());
+                } catch (ExecutionException e) {
+                    LOG.error("EosChangeListener: Inventory oper read failed for {}, Reason: {}",
+                            ncIdentifier, e.getMessage());
+                }
+            }
+        }
+    }
+
+
+    ////////private method area////////
+    private List<StateTunnelList> getUnknownTunnelsFromDefaultOper() {
+        List<StateTunnelList> unknownTunnels = new ArrayList<>();
+        Collection<StateTunnelList> tunnelList = tunnelStateCache.getAllPresent();
+        if (!tunnelList.isEmpty()) {
+            for (StateTunnelList stateTunnelEntry : tunnelList) {
+                if (stateTunnelEntry.getOperState().equals(TunnelOperStatus.Unknown)) {
+                    unknownTunnels.add(stateTunnelEntry);
+                }
+            }
+        } else {
+            LOG.debug("ITM oper DS is empty.");
+            EVENT_LOGGER.debug("ITM-EosChange, oper DS is empty.");
+        }
+        return unknownTunnels;
+    }
+
+    private ConcurrentMap<String, String> prepareKeyForInventoryOper(List<StateTunnelList> unknownTunnels) {
+        ConcurrentMap<String, String> tunnelNameToNodeConnectorIdValue = new ConcurrentHashMap<>();
+        for (StateTunnelList unknownTunnel:unknownTunnels) {
+            tunnelNameToNodeConnectorIdValue.put(unknownTunnel.getTunnelInterfaceName(),
+                    "openflow:" + unknownTunnel.getSrcInfo().getTepDeviceId() + ":" + unknownTunnel.getPortNumber());
+        }
+        return tunnelNameToNodeConnectorIdValue;
+    }
+
+    private void updateTunnelState(String interfaceName, TypedWriteTransaction<Operational> writeOnlyTx) {
+        StateTunnelListBuilder stateTnlBuilder = new StateTunnelListBuilder();
+        stateTnlBuilder.withKey(new StateTunnelListKey(interfaceName));
+        stateTnlBuilder.setTunnelState(true);
+        stateTnlBuilder.setOperState(TunnelOperStatus.Up);
+        InstanceIdentifier<StateTunnelList> tnlStateId = ItmUtils.buildStateTunnelListId(
+                new StateTunnelListKey(interfaceName));
+        writeOnlyTx.merge(tnlStateId, stateTnlBuilder.build());
+    }
+}