vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Strings;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import java.util.ArrayList;
  17 import java.util.Collections;
  18 import java.util.HashMap;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Objects;
  23 import java.util.Optional;
  24 import java.util.Set;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.Future;
  28 import java.util.concurrent.locks.ReentrantLock;
  29 import javax.annotation.PreDestroy;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  35 import org.opendaylight.genius.mdsalutil.NwConstants;
  36 import org.opendaylight.genius.utils.JvmGlobalLocks;
  37 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  38 import org.opendaylight.infrautils.utils.concurrent.Executors;
  39 import org.opendaylight.mdsal.binding.api.DataBroker;
  40 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  41 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  42 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  43 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  44 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  45 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  46 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  47 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  50 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.ParentRefs;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetDpnInterfaceListInputBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetDpnInterfaceListOutput;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.get.dpn._interface.list.output.Interfaces;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeExternal;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeHwvtep;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TepTypeInternal;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.extraroute.rds.map.extraroute.rds.DestPrefixes;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
  77 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  78 import org.opendaylight.yangtools.yang.common.RpcResult;
  79 import org.opendaylight.yangtools.yang.common.Uint32;
  80 import org.opendaylight.yangtools.yang.common.Uint64;
  81 import org.slf4j.Logger;
  82 import org.slf4j.LoggerFactory;
  83
  84 @Singleton
  85 public class TunnelInterfaceStateListener extends AbstractAsyncDataTreeChangeListener<StateTunnelList> {
  86
  87     private static final Logger LOG = LoggerFactory.getLogger(TunnelInterfaceStateListener.class);
  88
  89     private final DataBroker dataBroker;
  90     private final ManagedNewTransactionRunner txRunner;
  91     private final IFibManager fibManager;
  92     private final OdlInterfaceRpcService intfRpcService;
  93     private final VpnInterfaceManager vpnInterfaceManager;
  94     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
  95     private final JobCoordinator jobCoordinator;
  96     private final VpnUtil vpnUtil;
  97     private static final int RETRY_COUNT = 3;
  98
  99     protected enum UpdateRouteAction {
 100         ADVERTISE_ROUTE, WITHDRAW_ROUTE
 101     }
 102
 103     protected enum TunnelAction {
 104         TUNNEL_EP_ADD,
 105         TUNNEL_EP_DELETE,
 106         TUNNEL_EP_UPDATE
 107     }
 108
 109     /**
 110      * Responsible for listening to tunnel interface state change.
 111      * @param dataBroker Data Broker
 112      * @param fibManager FIB APIs
 113      * @param ifaceMgrRpcService Interface Manager RPC
 114      * @param vpnInterfaceManager Vpn Interface APIs
 115      * @param vpnSubnetRouteHandler Subnet-Route APIs
 116      * @param jobCoordinator Key based job serialization mechanism
 117      * @param vpnUtil Vpn Utility
 118      */
 119     @Inject
 120     public TunnelInterfaceStateListener(final DataBroker dataBroker,
 121                                         final IFibManager fibManager,
 122                                         final OdlInterfaceRpcService ifaceMgrRpcService,
 123                                         final VpnInterfaceManager vpnInterfaceManager,
 124                                         final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 125                                         final JobCoordinator jobCoordinator,
 126                                         VpnUtil vpnUtil) {
 127         super(dataBroker, LogicalDatastoreType.OPERATIONAL,
 128                 InstanceIdentifier.create(TunnelsState.class).child(StateTunnelList.class),
 129                 Executors.newListeningSingleThreadExecutor("TunnelInterfaceStateListener", LOG));
 130         this.dataBroker = dataBroker;
 131         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 132         this.fibManager = fibManager;
 133         this.intfRpcService = ifaceMgrRpcService;
 134         this.vpnInterfaceManager = vpnInterfaceManager;
 135         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 136         this.jobCoordinator = jobCoordinator;
 137         this.vpnUtil = vpnUtil;
 138         start();
 139     }
 140
 141     public void start() {
 142         LOG.info("{} start", getClass().getSimpleName());
 143     }
 144
 145     @Override
 146     @PreDestroy
 147     public void close() {
 148         super.close();
 149         Executors.shutdownAndAwaitTermination(getExecutorService());
 150     }
 151
 152
 153     @Override
 154     public void remove(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList del) {
 155         LOG.trace("remove: Tunnel deletion---- {}", del);
 156         if (isGreTunnel(del)) {
 157             programDcGwLoadBalancingGroup(del, NwConstants.MOD_FLOW, false);
 158         }
 159         handleTunnelEventForDPN(del, TunnelAction.TUNNEL_EP_DELETE);
 160     }
 161
 162     @Override
 163     public void update(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList original,
 164                           StateTunnelList update) {
 165         LOG.trace("update: Tunnel updation---- {}", update);
 166         LOG.info("update: ITM Tunnel {} of type {} state event changed from :{} to :{}",
 167                 update.getTunnelInterfaceName(),
 168                 fibManager.getTransportTypeStr(update.getTransportType().toString()),
 169                 original.getOperState(), update.getOperState());
 170         TunnelOperStatus tunOpStatus = update.getOperState();
 171         if (tunOpStatus != TunnelOperStatus.Down && tunOpStatus != TunnelOperStatus.Up) {
 172             LOG.info("update: Returning from unsupported tunnelOperStatus {} for tunnel interface {}", tunOpStatus,
 173                     update.getTunnelInterfaceName());
 174             return;
 175         }
 176         boolean isTunnelUp = TunnelOperStatus.Up == update.getOperState();
 177         if (isGreTunnel(update)) {
 178             programDcGwLoadBalancingGroup(update, NwConstants.MOD_FLOW, isTunnelUp);
 179         }
 180
 181         //Remove the corresponding nexthop from the routepath under extraroute in fibentries.
 182         Uint64 srcDpnId = Uint64.valueOf(update.getSrcInfo().getTepDeviceId()).intern();
 183         String srcTepIp = update.getSrcInfo().getTepIp().stringValue();
 184         List<VpnInstanceOpDataEntry> vpnInstanceOpData = vpnUtil.getAllVpnInstanceOpData();
 185         if (vpnInstanceOpData == null) {
 186             LOG.trace("update: No vpnInstanceOpdata present");
 187             return;
 188         }
 189         vpnInstanceOpData.stream()
 190                 .filter(opData -> opData.getVpnToDpnList() != null
 191                         && opData.getVpnToDpnList().stream().anyMatch(
 192                             vpnToDpn -> Objects.equals(vpnToDpn.getDpnId(), srcDpnId)))
 193                 .forEach(opData -> {
 194                     List<DestPrefixes> prefixes = VpnExtraRouteHelper.getExtraRouteDestPrefixes(dataBroker,
 195                             opData.getVpnId());
 196                     prefixes.forEach(destPrefix -> {
 197                         VrfEntry vrfEntry = vpnUtil.getVrfEntry(opData.getVrfId(),
 198                                 destPrefix.getDestPrefix());
 199                         if (vrfEntry == null || vrfEntry.getRoutePaths() == null) {
 200                             return;
 201                         }
 202                         List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
 203                         routePaths.forEach(routePath -> {
 204                             if (Objects.equals(routePath.getNexthopAddress(), srcTepIp)) {
 205                                 String prefix = destPrefix.getDestPrefix();
 206                                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(opData.getVpnInstanceName(),
 207                                         prefix);
 208                                 // FIXME: separate out to somehow?
 209                                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 210                                 lock.lock();
 211                                 try {
 212                                     fibManager.refreshVrfEntry(opData.getVrfId(), prefix);
 213                                 } finally {
 214                                     lock.unlock();
 215                                 }
 216                             }
 217                         });
 218                     });
 219                 });
 220     }
 221
 222     @Override
 223     public void add(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList add) {
 224         LOG.trace("add: Tunnel addition---- {}", add);
 225         TunnelOperStatus tunOpStatus = add.getOperState();
 226         if (tunOpStatus != TunnelOperStatus.Down && tunOpStatus != TunnelOperStatus.Up) {
 227             LOG.info("add: Returning from unsupported tunnelOperStatus {} for tunnel interface {}", tunOpStatus,
 228                     add.getTunnelInterfaceName());
 229             return;
 230         }
 231         if (tunOpStatus != TunnelOperStatus.Up) {
 232             LOG.error("add: Tunnel {} is not yet UP.", add.getTunnelInterfaceName());
 233         }
 234         boolean isTunnelUp = TunnelOperStatus.Up == add.getOperState();
 235         if (isGreTunnel(add)) {
 236             programDcGwLoadBalancingGroup(add, NwConstants.ADD_FLOW, isTunnelUp);
 237         }
 238         LOG.info("add: ITM Tunnel ,type {} ,added between src: {} and dest: {}",
 239                 fibManager.getTransportTypeStr(add.getTransportType() != null
 240                         ? add.getTransportType().toString() : "Invalid"),
 241                 add.getSrcInfo() != null ? add.getSrcInfo().getTepDeviceId() : "0",
 242                 add.getDstInfo() != null ? add.getDstInfo().getTepDeviceId() : "0");
 243         handleTunnelEventForDPN(add, TunnelAction.TUNNEL_EP_ADD);
 244     }
 245
 246     public enum TunnelEventProcessingMethod {
 247         POPULATESUBNETS(0), MANAGEREMOTEROUTES(1);
 248
 249         private final int method;
 250
 251         TunnelEventProcessingMethod(int id) {
 252             this.method = id;
 253         }
 254
 255         public int getValue() {
 256             return method;
 257         }
 258     }
 259
 260     // TODO Clean up the exception handling
 261     @SuppressWarnings("checkstyle:IllegalCatch")
 262     private void handleTunnelEventForDPN(StateTunnelList stateTunnelList, TunnelAction tunnelAction) {
 263         final Uint64 srcDpnId = stateTunnelList.getSrcInfo() != null
 264                 ? Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern() : Uint64.ZERO;
 265         final String srcTepIp = stateTunnelList.getSrcInfo() != null
 266                 ? stateTunnelList.getSrcInfo().getTepIp().stringValue() : "0";
 267         String destTepIp = stateTunnelList.getDstInfo() != null
 268                 ? stateTunnelList.getDstInfo().getTepIp().stringValue() : "0";
 269         String rd;
 270         Uint64 remoteDpnId = null;
 271         boolean isTepDeletedOnDpn = false;
 272
 273         LOG.info("handleTunnelEventForDPN: Handle tunnel event for srcDpn {} SrcTepIp {} DestTepIp {} ",
 274                 srcDpnId, srcTepIp, destTepIp);
 275         int tunTypeVal = getTunnelType(stateTunnelList);
 276         LOG.trace("handleTunnelEventForDPN: tunTypeVal is {}", tunTypeVal);
 277         try {
 278             if (tunnelAction == TunnelAction.TUNNEL_EP_ADD) {
 279                 LOG.info("handleTunnelEventForDPN: Tunnel ADD event received for Dpn {} VTEP Ip {} destTepIp {}",
 280                         srcDpnId, srcTepIp, destTepIp);
 281                 if (isTunnelInLogicalGroup(stateTunnelList)) {
 282                     return;
 283                 }
 284             } else if (tunnelAction == TunnelAction.TUNNEL_EP_DELETE) {
 285                 LOG.info("handleTunnelEventForDPN: Tunnel DELETE event received for Dpn {} VTEP Ip {} DestTepIp {}",
 286                         srcDpnId, srcTepIp, destTepIp);
 287                 // When tunnel EP is deleted on a DPN , VPN gets two deletion event.
 288                 // One for a DPN on which tunnel EP was deleted and another for other-end DPN.
 289                 // Update the adj for the vpninterfaces for a DPN on which TEP is deleted.
 290                 // Update the adj & VRF for the vpninterfaces for a DPN on which TEP is deleted.
 291                 // Dont update the adj & VRF for vpninterfaces for a DPN on which TEP is not deleted.
 292                 String endpointIpForDPN;
 293                 try {
 294                     endpointIpForDPN = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, srcDpnId);
 295                 } catch (Exception e) {
 296                     LOG.error("handleTunnelEventForDPN: Unable to resolve endpoint IP for srcDpn {}", srcDpnId);
 297                     /* this dpn does not have the VTEP */
 298                     endpointIpForDPN = null;
 299                 }
 300
 301                 if (endpointIpForDPN == null) {
 302                     LOG.info("handleTunnelEventForDPN: Tunnel TEP is deleted on Dpn {} VTEP Ip {} destTepIp {}",
 303                             srcDpnId, srcTepIp, destTepIp);
 304                     isTepDeletedOnDpn = true;
 305                 }
 306             }
 307
 308             // Get the list of VpnInterfaces from Intf Mgr for a SrcDPN on which TEP is added/deleted
 309             Future<RpcResult<GetDpnInterfaceListOutput>> result;
 310             List<Interfaces> srcDpninterfacelist = new ArrayList<>();
 311             List<Interfaces> destDpninterfacelist = new ArrayList<>();
 312             try {
 313                 result = intfRpcService.getDpnInterfaceList(
 314                         new GetDpnInterfaceListInputBuilder().setDpid(srcDpnId).build());
 315                 RpcResult<GetDpnInterfaceListOutput> rpcResult = result.get();
 316                 if (!rpcResult.isSuccessful()) {
 317                     LOG.error("handleTunnelEventForDPN: RPC Call to GetDpnInterfaceList for srcDpnid {} srcTepIp {}"
 318                                     + " destTepIP {} returned with Errors {}", srcDpnId, srcTepIp, destTepIp,
 319                             rpcResult.getErrors());
 320                 } else {
 321                     srcDpninterfacelist = rpcResult.getResult().nonnullInterfaces();
 322                 }
 323             } catch (Exception e) {
 324                 LOG.error("handleTunnelEventForDPN: Exception when querying for GetDpnInterfaceList for srcDpnid {}"
 325                         + " srcTepIp {} destTepIp {}", srcDpnId, srcTepIp, destTepIp, e);
 326             }
 327             // Get the list of VpnInterfaces from Intf Mgr for a destDPN only for internal tunnel.
 328             if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
 329                 remoteDpnId = Uint64.valueOf(stateTunnelList.getDstInfo() != null
 330                         ? stateTunnelList.getDstInfo().getTepDeviceId() : "0").intern();
 331                 try {
 332                     result = intfRpcService.getDpnInterfaceList(
 333                             new GetDpnInterfaceListInputBuilder().setDpid(remoteDpnId).build());
 334                     RpcResult<GetDpnInterfaceListOutput> rpcResult = result.get();
 335                     if (!rpcResult.isSuccessful()) {
 336                         LOG.error("handleTunnelEventForDPN: RPC Call to GetDpnInterfaceList for remoteDpnid {}"
 337                                         + " srcTepIP {} destTepIp {} returned with Errors {}", remoteDpnId, srcTepIp,
 338                                 destTepIp, rpcResult.getErrors());
 339                     } else {
 340                         destDpninterfacelist = rpcResult.getResult().nonnullInterfaces();
 341                     }
 342                 } catch (Exception e) {
 343                     LOG.error("handleTunnelEventForDPN: Exception when querying for GetDpnInterfaceList"
 344                                     + " for remoteDpnid {} srcTepIp {} destTepIp {}", remoteDpnId,
 345                             srcTepIp, destTepIp, e);
 346                 }
 347             }
 348
 349             /*
 350              * Iterate over the list of VpnInterface for a SrcDpn on which TEP is added or deleted and read the adj.
 351              * Update the adjacencies with the updated nexthop.
 352              */
 353             List<Uuid> subnetList = new ArrayList<>();
 354             Map<Uint32, String> vpnIdRdMap = new HashMap<>();
 355             Set<String> listVpnName = new HashSet<>();
 356
 357             for (Interfaces interfaces : srcDpninterfacelist) {
 358                 if (!L2vlan.class.equals(interfaces.getInterfaceType())) {
 359                     LOG.info("handleTunnelEventForDPN: Interface {} not of type L2Vlan", interfaces.getInterfaceName());
 360                     continue;
 361                 }
 362                 String intfName = interfaces.getInterfaceName();
 363                 VpnInterface vpnInterface =
 364                      vpnUtil.getConfiguredVpnInterface(intfName);
 365                 if (vpnInterface != null) {
 366                     listVpnName.addAll(VpnHelper
 367                         .getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 368                     handleTunnelEventForDPNVpn(stateTunnelList, vpnIdRdMap,
 369                             tunnelAction, isTepDeletedOnDpn,
 370                             subnetList, TunnelEventProcessingMethod.POPULATESUBNETS,
 371                             vpnInterface);
 372                 }
 373             }
 374             /*
 375              * Iterate over the list of VpnInterface for destDPN and get the prefix .
 376              * Create remote rule for each of those prefix on srcDPN.
 377              */
 378             for (Interfaces interfaces : destDpninterfacelist) {
 379                 if (!L2vlan.class.equals(interfaces.getInterfaceType())) {
 380                     LOG.info("handleTunnelEventForDPN: Interface {} not of type L2Vlan", interfaces.getInterfaceName());
 381                     continue;
 382                 }
 383                 String intfName = interfaces.getInterfaceName();
 384                 VpnInterface vpnInterface =
 385                         vpnUtil.getConfiguredVpnInterface(intfName);
 386                 if (vpnInterface != null) {
 387                     handleTunnelEventForDPNVpn(stateTunnelList, vpnIdRdMap,
 388                             tunnelAction, isTepDeletedOnDpn,
 389                             subnetList, TunnelEventProcessingMethod.MANAGEREMOTEROUTES,
 390                             vpnInterface);
 391                 }
 392             }
 393
 394             //Iterate over the VpnId-to-Rd map.
 395             for (Map.Entry<Uint32, String> entry : vpnIdRdMap.entrySet()) {
 396                 Uint32 vpnId = entry.getKey();
 397                 rd = entry.getValue();
 398                 if (tunnelAction == TunnelAction.TUNNEL_EP_ADD
 399                     && tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
 400                     fibManager.populateExternalRoutesOnDpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
 401                 } else if (tunnelAction == TunnelAction.TUNNEL_EP_DELETE
 402                     && tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue()) {
 403                     fibManager.cleanUpExternalRoutesOnDpn(srcDpnId, vpnId, rd, srcTepIp, destTepIp);
 404                 }
 405             }
 406             if (listVpnName.size() >= 1) {
 407                 if (tunnelAction == TunnelAction.TUNNEL_EP_ADD) {
 408                     for (Uuid subnetId : subnetList) {
 409                         // Populate the List of subnets
 410                         vpnSubnetRouteHandler.updateSubnetRouteOnTunnelUpEvent(subnetId, srcDpnId);
 411                     }
 412                 }
 413                 if (tunnelAction == TunnelAction.TUNNEL_EP_DELETE && isTepDeletedOnDpn) {
 414                     for (Uuid subnetId : subnetList) {
 415                         // Populate the List of subnets
 416                         vpnSubnetRouteHandler.updateSubnetRouteOnTunnelDownEvent(subnetId, srcDpnId);
 417                     }
 418                 }
 419             }
 420             /*
 421              * Program the BGP routes of all the VPNs which have footprint on the source DPN.
 422              *
 423              * DC-GW LB groups are programmed in DJC Jobs, so DJC with same key is used here to make sure
 424              * groups are programmed first, then only BGP routes are programmed.
 425              */
 426             jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(srcDpnId), () -> {
 427                 listVpnName.forEach(vpnName -> {
 428                     Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 429                     final String vrfId = vpnIdRdMap.get(vpnId);
 430                     if ((tunnelAction == TunnelAction.TUNNEL_EP_ADD)
 431                             && (tunTypeVal == VpnConstants.ITMTunnelLocType.External.getValue())) {
 432                         fibManager.populateExternalRoutesOnDpn(srcDpnId, vpnId, vrfId,
 433                                 srcTepIp, destTepIp);
 434                     }
 435                 });
 436                 return Collections.emptyList();
 437             },RETRY_COUNT);
 438         } catch (RuntimeException e) {
 439             LOG.error("handleTunnelEventForDpn: Unable to handle the tunnel event for srcDpnId {} srcTepIp {}"
 440                     + " remoteDpnId {} destTepIp {}", srcDpnId, srcTepIp, remoteDpnId, destTepIp, e);
 441         }
 442     }
 443
 444     // TODO Clean up the exception handling
 445     @SuppressWarnings("checkstyle:IllegalCatch")
 446     private void handleTunnelEventForDPNVpn(StateTunnelList stateTunnelList,
 447                                             Map<Uint32, String> vpnIdRdMap, TunnelAction tunnelAction,
 448                                             boolean isTepDeletedOnDpn, List<Uuid> subnetList,
 449                                             TunnelEventProcessingMethod method,
 450                                             VpnInterface cfgVpnInterface) {
 451         String rd;
 452         String intfName = cfgVpnInterface.getName();
 453         final Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo() != null
 454                 ? stateTunnelList.getSrcInfo().getTepDeviceId() : "0").intern();
 455         String destTepIp = stateTunnelList.getDstInfo() != null ? stateTunnelList.getDstInfo().getTepIp().stringValue()
 456                 : null;
 457         String srcTepIp = stateTunnelList.getSrcInfo() != null ? stateTunnelList.getSrcInfo().getTepIp().stringValue()
 458                 : null;
 459         int tunTypeVal = getTunnelType(stateTunnelList);
 460         Uint64 remoteDpnId = null;
 461         if (tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
 462             remoteDpnId = Uint64.valueOf(stateTunnelList.getDstInfo() != null
 463                     ? stateTunnelList.getDstInfo().getTepDeviceId() : "0").intern();
 464         }
 465         if (cfgVpnInterface.getVpnInstanceNames() == null) {
 466             LOG.warn("handleTunnelEventForDpn: no vpnName found for interface {}", intfName);
 467             return;
 468         }
 469         try {
 470             for (VpnInstanceNames vpnInstance : cfgVpnInterface.getVpnInstanceNames()) {
 471                 String vpnName = vpnInstance.getVpnName();
 472                 if (method == TunnelEventProcessingMethod.POPULATESUBNETS) {
 473                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = vpnUtil
 474                             .getVpnInterfaceOpDataEntry(intfName, vpnName);
 475                     if (opVpnInterface.isPresent()) {
 476                         VpnInterfaceOpDataEntry vpnInterface  = opVpnInterface.get();
 477                         jobCoordinator.enqueueJob("VPNINTERFACE-" + intfName,
 478                                 new UpdateVpnInterfaceOnTunnelEvent(tunnelAction,
 479                                         vpnInterface,
 480                                         stateTunnelList,
 481                                         isTepDeletedOnDpn));
 482
 483                         // Populate the List of subnets
 484                         InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 485                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 486                                         new PortOpDataEntryKey(intfName)).build();
 487                         Optional<PortOpDataEntry> optionalPortOp =
 488                                 SingleTransactionDataBroker.syncReadOptional(dataBroker,
 489                                         LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 490                         if (optionalPortOp.isPresent()) {
 491                             List<Uuid> subnetIdList = optionalPortOp.get().getSubnetIds();
 492                             if (subnetIdList != null) {
 493                                 for (Uuid subnetId : subnetIdList) {
 494                                     if (!subnetList.contains(subnetId)) {
 495                                         subnetList.add(subnetId);
 496                                     }
 497                                 }
 498                             }
 499                         }
 500                         //Populate the map for VpnId-to-Rd
 501                         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 502                         rd = vpnUtil.getVpnRd(vpnName);
 503                         vpnIdRdMap.put(vpnId, rd);
 504                     }
 505                 } else if (method == TunnelEventProcessingMethod.MANAGEREMOTEROUTES) {
 506                     Optional<VpnInterfaceOpDataEntry> opVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(intfName,
 507                             vpnName);
 508                     if (opVpnInterface.isPresent()) {
 509                         VpnInterfaceOpDataEntry vpnInterface  = opVpnInterface.get();
 510                         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 511                         List<Adjacency> adjList =
 512                             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency()
 513                                 : emptyList();
 514                         String prefix = null;
 515                         Uint32 vpnId = vpnUtil.getVpnId(vpnInterface.getVpnInstanceName());
 516                         if (vpnIdRdMap.containsKey(vpnId)) {
 517                             rd = vpnIdRdMap.get(vpnId);
 518                             LOG.info("handleTunnelEventForDPN: Remote DpnId {} VpnId {} rd {} VpnInterface {}"
 519                                     + " srcTepIp {} destTepIp {}", remoteDpnId, vpnId, rd , vpnInterface, srcTepIp,
 520                                     destTepIp);
 521                             for (Adjacency adj : adjList) {
 522                                 prefix = adj.getIpAddress();
 523                                 Uint32 label = adj.getLabel();
 524                                 if (tunnelAction == TunnelAction.TUNNEL_EP_ADD
 525                                         && tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
 526                                     fibManager.manageRemoteRouteOnDPN(true, srcDpnId, vpnId, rd, prefix, destTepIp,
 527                                             label);
 528                                 }
 529                                 if (tunnelAction == TunnelAction.TUNNEL_EP_DELETE
 530                                         && tunTypeVal == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
 531                                     fibManager.manageRemoteRouteOnDPN(false, srcDpnId, vpnId, rd, prefix, destTepIp,
 532                                             label);
 533                                 }
 534                             }
 535                         }
 536                     }
 537                 }
 538             }
 539         } catch (InterruptedException | ExecutionException e) {
 540             LOG.error("handleTunnelEventForDPN: Failed to read data store for interface {} srcDpn {} srcTep {} "
 541                     + "dstTep {}", intfName, srcDpnId, srcTepIp, destTepIp);
 542         }
 543     }
 544
 545     private class UpdateVpnInterfaceOnTunnelEvent implements Callable {
 546         private final VpnInterfaceOpDataEntry vpnInterface;
 547         private final StateTunnelList stateTunnelList;
 548         private final TunnelAction tunnelAction;
 549         private final boolean isTepDeletedOnDpn;
 550
 551         UpdateVpnInterfaceOnTunnelEvent(TunnelAction tunnelAction,
 552                                         VpnInterfaceOpDataEntry vpnInterface,
 553                                         StateTunnelList stateTunnelList,
 554                                         boolean isTepDeletedOnDpn) {
 555             this.stateTunnelList = stateTunnelList;
 556             this.vpnInterface = vpnInterface;
 557             this.tunnelAction = tunnelAction;
 558             this.isTepDeletedOnDpn = isTepDeletedOnDpn;
 559         }
 560
 561         @Override
 562         public List<ListenableFuture<Void>> call() {
 563             List<ListenableFuture<Void>> futures = new ArrayList<>(2);
 564             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx ->
 565                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 566                     if (tunnelAction == TunnelAction.TUNNEL_EP_ADD) {
 567                         vpnInterfaceManager.updateVpnInterfaceOnTepAdd(vpnInterface, stateTunnelList, confTx, operTx);
 568                     }
 569
 570                     if (tunnelAction == TunnelAction.TUNNEL_EP_DELETE && isTepDeletedOnDpn) {
 571                         vpnInterfaceManager.updateVpnInterfaceOnTepDelete(vpnInterface, stateTunnelList, confTx,
 572                             operTx);
 573                     }
 574                 }))));
 575             return futures;
 576         }
 577     }
 578
 579     private int getTunnelType(StateTunnelList stateTunnelList) {
 580         int tunTypeVal = 0;
 581         if (stateTunnelList.getDstInfo() == null) {
 582             return VpnConstants.ITMTunnelLocType.Invalid.getValue();
 583         }
 584         if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeInternal.class) {
 585             tunTypeVal = VpnConstants.ITMTunnelLocType.Internal.getValue();
 586         } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeExternal.class) {
 587             tunTypeVal = VpnConstants.ITMTunnelLocType.External.getValue();
 588         } else if (stateTunnelList.getDstInfo().getTepDeviceType() == TepTypeHwvtep.class) {
 589             tunTypeVal = VpnConstants.ITMTunnelLocType.Hwvtep.getValue();
 590         } else {
 591             tunTypeVal = VpnConstants.ITMTunnelLocType.Invalid.getValue();
 592         }
 593         return tunTypeVal;
 594     }
 595
 596     private boolean isGreTunnel(StateTunnelList del) {
 597         return del.getTransportType() == TunnelTypeMplsOverGre.class;
 598     }
 599
 600     private void programDcGwLoadBalancingGroup(StateTunnelList tunnelState, int addOrRemove, boolean isTunnelUp) {
 601         IpAddress dcGwIp = tunnelState.getDstInfo().getTepIp();
 602         String dcGwIpAddress = String.valueOf(dcGwIp.stringValue());
 603         Uint64 dpId = Uint64.valueOf(tunnelState.getSrcInfo().getTepDeviceId()).intern();
 604         fibManager.programDcGwLoadBalancingGroup(dpId, dcGwIpAddress, addOrRemove, isTunnelUp,
 605                 tunnelState.getTransportType());
 606     }
 607
 608     private boolean isTunnelInLogicalGroup(StateTunnelList stateTunnelList) {
 609         String ifaceName = stateTunnelList.getTunnelInterfaceName();
 610         if (getTunnelType(stateTunnelList) == VpnConstants.ITMTunnelLocType.Internal.getValue()) {
 611             Interface configIface = InterfaceUtils.getInterface(dataBroker, stateTunnelList.getTunnelInterfaceName());
 612             IfTunnel ifTunnel = configIface != null ? configIface.augmentation(IfTunnel.class) : null;
 613             if (ifTunnel != null && ifTunnel.getTunnelInterfaceType().isAssignableFrom(TunnelTypeVxlan.class)) {
 614                 ParentRefs refs = configIface.augmentation(ParentRefs.class);
 615                 if (refs != null && !Strings.isNullOrEmpty(refs.getParentInterface())) {
 616                     return true; //multiple VxLAN tunnels enabled, i.e. only logical tunnel should be treated
 617                 }
 618             }
 619         }
 620         LOG.trace("isTunnelInLogicalGroup: MULTIPLE_VxLAN_TUNNELS: ignoring the tunnel event for {}", ifaceName);
 621         return false;
 622     }
 623 }