vpnservice/natservice/natservice-impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.collect.Sets;
  12
  13 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  14 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  15 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  16 import org.opendaylight.genius.mdsalutil.*;
  17 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  18 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  19 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  20 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.*;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.IpMapping;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.IpMappingKey;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  57 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  58 import org.opendaylight.yangtools.yang.common.RpcResult;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 import java.math.BigInteger;
  63 import java.util.ArrayList;
  64 import java.util.List;
  65 import java.util.Set;
  66 import java.util.concurrent.ExecutionException;
  67 import java.util.concurrent.Future;
  68 import java.util.HashMap;
  69
  70 public class NaptSwitchHA {
  71     private static final Logger LOG = LoggerFactory.getLogger(NaptSwitchHA.class);
  72     private final DataBroker dataBroker;
  73     private final IMdsalApiManager mdsalManager;
  74     private final ItmRpcService itmManager;
  75     private final OdlInterfaceRpcService interfaceManager;
  76     private final IdManagerService idManager;
  77     private final NAPTSwitchSelector naptSwitchSelector;
  78     private final ExternalRoutersListener externalRouterListener;
  79     private final IBgpManager bgpManager;
  80     private final VpnRpcService vpnService;
  81     private final FibRpcService fibService;
  82     private List<String> externalIpsCache;
  83     private HashMap<String,Long> externalIpsLabel;
  84
  85     public NaptSwitchHA(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  86                         final ExternalRoutersListener externalRouterListener,
  87                         final ItmRpcService itmManager,
  88                         final OdlInterfaceRpcService interfaceManager,
  89                         final IdManagerService idManager,
  90                         final NAPTSwitchSelector naptSwitchSelector,
  91                         final IBgpManager bgpManager,
  92                         final VpnRpcService vpnService,
  93                         final FibRpcService fibService) {
  94         this.dataBroker = dataBroker;
  95         this.mdsalManager = mdsalManager;
  96         this.externalRouterListener = externalRouterListener;
  97         this.itmManager = itmManager;
  98         this.interfaceManager = interfaceManager;
  99         this.idManager = idManager;
 100         this.naptSwitchSelector = naptSwitchSelector;
 101         this.bgpManager = bgpManager;
 102         this.vpnService = vpnService;
 103         this.fibService =fibService;
 104     }
 105
 106     /* This method checks the switch that gone down is a NaptSwitch for a router.
 107        If it is a NaptSwitch
 108           1) selects new NAPT switch
 109           2) installs nat flows in new NAPT switch
 110           table 21(FIB)->26(PSNAT)->group(resubmit/napttunnel)->36(Terminating)->46(outbound)->47(resubmit)->21
 111           3) modify the group and miss entry flow in other vSwitches pointing to newNaptSwitch
 112           4) Remove nat flows in oldNaptSwitch
 113      */
 114     /*public void handleNaptSwitchDown(BigInteger dpnId){
 115
 116         LOG.debug("handleNaptSwitchDown method is called with dpnId {}",dpnId);
 117         BigInteger naptSwitch;
 118         try {
 119             NaptSwitches naptSwitches = NatUtil.getNaptSwitch(dataBroker);
 120             if (naptSwitches == null || naptSwitches.getRouterToNaptSwitch() == null || naptSwitches.getRouterToNaptSwitch().isEmpty()) {
 121                 LOG.debug("NaptSwitchDown: NaptSwitch is not allocated for none of the routers");
 122                 return;
 123             }
 124             for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.getRouterToNaptSwitch()) {
 125                 String routerName = routerToNaptSwitch.getRouterName();
 126                 naptSwitch = routerToNaptSwitch.getPrimarySwitchId();
 127                 boolean naptStatus = isNaptSwitchDown(routerName,dpnId,naptSwitch);
 128                 if (!naptStatus) {
 129                     LOG.debug("NaptSwitchDown: Switch with DpnId {} is not naptSwitch for router {}",
 130                             dpnId, routerName);
 131                 } else {
 132                     removeSnatFlowsInOldNaptSwitch(routerName,naptSwitch);
 133                     return;
 134                 }
 135             }
 136         } catch (Exception ex) {
 137             LOG.error("Exception in handleNaptSwitchDown method {}",ex);
 138         }
 139     }*/
 140
 141     protected void removeSnatFlowsInOldNaptSwitch(String routerName, BigInteger naptSwitch,HashMap<String,Long> externalIpmap) {
 142         externalIpsLabel = externalIpmap;
 143         //remove SNAT flows in old NAPT SWITCH
 144         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 145         if (routerId == NatConstants.INVALID_ID) {
 146             LOG.error("Invalid routerId returned for routerName {}",routerName);
 147             return;
 148         }
 149
 150         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table
 151         String tsFlowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
 152         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
 153
 154         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}"
 155                 ,NwConstants.INTERNAL_TUNNEL_TABLE, naptSwitch, routerId);
 156         mdsalManager.removeFlow(tsNatFlowEntity);
 157
 158         //Remove the Outbound flow entry which forwards the packet to Outbound NAPT Table
 159         String outboundNatFlowRef = externalRouterListener.getFlowRefOutbound(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
 160         FlowEntity outboundNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch,
 161                 NwConstants.OUTBOUND_NAPT_TABLE, outboundNatFlowRef);
 162         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}"
 163                 ,NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId);
 164         mdsalManager.removeFlow(outboundNatFlowEntity);
 165
 166         //Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for inbound traffic matching on the router ID.
 167         String naptPFibflowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, routerId);
 168         FlowEntity naptPFibFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE, naptPFibflowRef);
 169         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}",
 170                 NwConstants.NAPT_PFIB_TABLE, naptSwitch, routerId);
 171         mdsalManager.removeFlow(naptPFibFlowEntity);
 172
 173         //Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for outbound traffic matching on the vpn ID.
 174         boolean switchSharedByRouters = false;
 175         Uuid extNetworkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 176         if (extNetworkId != null) {
 177             List<String> routerNamesAssociated = getRouterIdsForExtNetwork(extNetworkId);
 178             if (routerNamesAssociated != null) {
 179                 for (String routerNameAssociated : routerNamesAssociated) {
 180                     if (!routerNameAssociated.equals(routerName)) {
 181                         Long routerIdAssociated = NatUtil.getVpnId(dataBroker,routerNameAssociated);
 182                         BigInteger naptDpn = NatUtil.getPrimaryNaptfromRouterId(dataBroker,routerIdAssociated);
 183                         if (naptDpn != null && naptDpn.equals(naptSwitch)) {
 184                             LOG.debug("Napt switch {} is also acting as primary for router {}",routerIdAssociated);
 185                             switchSharedByRouters = true;
 186                             break;
 187                         }
 188                     }
 189                 }
 190                 if (!switchSharedByRouters) {
 191                     Long vpnId = getVpnIdForRouter(routerId);
 192                     if (vpnId != NatConstants.INVALID_ID) {
 193                         String naptFibflowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, vpnId);
 194                         FlowEntity naptFibFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE,naptFibflowRef);
 195                         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and vpnId {}",
 196                                 NwConstants.NAPT_PFIB_TABLE, naptSwitch, vpnId);
 197                         mdsalManager.removeFlow(naptFibFlowEntity);
 198                     } else {
 199                         LOG.error("Invalid vpnId retrieved for routerId {}",routerId);
 200                         return;
 201                     }
 202                 }
 203             }
 204         }
 205
 206         //Remove Fib entries,tables 20->44 ,36-> 44
 207         String vpnName = getExtNetworkVpnName(routerId);
 208         if (vpnName == null) {
 209             LOG.debug("Vpn is not associated to externalN/w of router {}",routerName);
 210         } else {
 211             if (externalIpsLabel != null) {
 212                 for (String externalIp : externalIpsLabel.keySet()) {
 213                     Long label = externalIpsLabel.get(externalIp);
 214                     externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerId, externalIp, vpnName,label);
 215                     LOG.debug("Successfully removed fib entries in old naptswitch {} for router {} and externalIps {} label {}",
 216                             naptSwitch, routerId,externalIp,label);
 217                 }
 218             } else {
 219                 List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 220                 if (externalIps != null) {
 221                     Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 222                     if (networkId != null) {
 223                         externalRouterListener.clearFibTsAndReverseTraffic(naptSwitch, routerId, networkId, externalIps, null);
 224                         LOG.debug("Successfully removed fib entries in old naptswitch {} for router {} with networkId {} and externalIps {}",
 225                                     naptSwitch,routerId,networkId,externalIps);
 226                     } else {
 227                         LOG.debug("External network not associated to router {}", routerId);
 228                     }
 229                 } else {
 230                     LOG.debug("ExternalIps not found for router {}",routerName);
 231                 }
 232             }
 233         }
 234
 235         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
 236         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
 237         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 238             LOG.debug("No Internal Ip Port mapping associated to router {}, no flows need to be removed in" +
 239                     "oldNaptSwitch {}", routerId, naptSwitch);
 240             return;
 241         }
 242         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
 243         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
 244         for(IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
 245             if (intextIpProtocolType.getIpPortMap() == null || intextIpProtocolType.getIpPortMap().isEmpty()) {
 246                 LOG.debug("No {} session associated to router {},no flows need to be removed in oldNaptSwitch {}",
 247                         intextIpProtocolType.getProtocol(),routerId,naptSwitch);
 248                 break;
 249             }
 250             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
 251             for(IpPortMap ipPortMap : ipPortMaps) {
 252                 String ipPortInternal = ipPortMap.getIpPortInternal();
 253                 String[] ipPortParts = ipPortInternal.split(":");
 254                 if(ipPortParts.length != 2) {
 255                     LOG.error("Unable to retrieve the Internal IP and port");
 256                     continue;
 257                 }
 258                 String internalIp = ipPortParts[0];
 259                 String internalPort = ipPortParts[1];
 260
 261                 //Build and remove flow in outbound NAPT table
 262                 String switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, String.valueOf(routerId),
 263                         internalIp, Integer.valueOf(internalPort));
 264                 FlowEntity outboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 265                         cookieSnatFlow, switchFlowRef);
 266
 267                 LOG.info("Remove the flow in table {} for old napt switch with the DPN ID {} and router ID {}",
 268                         NwConstants.OUTBOUND_NAPT_TABLE,naptSwitch, routerId);
 269                 mdsalManager.removeFlow(outboundNaptFlowEntity);
 270
 271                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
 272                 if (ipPortExternal == null) {
 273                     LOG.debug("External Ipport mapping not found for internalIp {} with port {} for router", internalIp,
 274                             internalPort, routerId);
 275                     continue;
 276                 }
 277                 String externalIp = ipPortExternal.getIpAddress();
 278                 int externalPort = ipPortExternal.getPortNum();
 279
 280                 //Build and remove flow in  inbound NAPT table
 281                 switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, String.valueOf(routerId),
 282                         externalIp, externalPort);
 283                 FlowEntity inboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 284                         cookieSnatFlow, switchFlowRef);
 285
 286                 LOG.info("Remove the flow in table {} for old napt switch with the DPN ID {} and router ID {}",
 287                         NwConstants.INBOUND_NAPT_TABLE,naptSwitch, routerId);
 288                 mdsalManager.removeFlow(inboundNaptFlowEntity);
 289             }
 290         }
 291
 292     }
 293
 294     private List<String> getRouterIdsForExtNetwork(Uuid extNetworkId) {
 295         List<String> routerUuidsAsString = new ArrayList<>();
 296         InstanceIdentifier<Networks> extNetwork = InstanceIdentifier.builder(ExternalNetworks.class).child
 297                 (Networks.class, new NetworksKey(extNetworkId)).build();
 298         Optional<Networks> extNetworkData = NatUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, extNetwork);
 299         if (extNetworkData.isPresent()) {
 300             List<Uuid> routerUuids= extNetworkData.get().getRouterIds();
 301             if (routerUuids != null){
 302                 for(Uuid routerUuid : routerUuids){
 303                     routerUuidsAsString.add(routerUuid.getValue());
 304                 }
 305             }
 306         }
 307         return routerUuidsAsString;
 308     }
 309
 310     public boolean isNaptSwitchDown(String routerName, BigInteger dpnId , BigInteger naptSwitch,Long routerVpnId,List<String> externalIpCache) {
 311         externalIpsCache = externalIpCache;
 312         if (!naptSwitch.equals(dpnId)) {
 313             LOG.debug("DpnId {} is not a naptSwitch {} for Router {}",dpnId, naptSwitch, routerName);
 314             return false;
 315         }
 316         LOG.debug("NaptSwitch {} is down for Router {}", naptSwitch, routerName);
 317         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 318         if (routerId == NatConstants.INVALID_ID) {
 319             LOG.error("Invalid routerId returned for routerName {}", routerName);
 320             return true;
 321         }
 322         //elect a new NaptSwitch
 323         naptSwitch = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 324         if (naptSwitch.equals(BigInteger.ZERO)) {
 325             LOG.info("No napt switch is elected since all the switches for router {} are down",routerName);
 326             boolean naptUpdatedStatus = updateNaptSwitch(routerName,naptSwitch);
 327             if(!naptUpdatedStatus) {
 328                 LOG.debug("Failed to update naptSwitch {} for router {} in ds", naptSwitch,routerName);
 329             }
 330             //clearBgpRoutes
 331             if (externalIpsCache != null) {
 332                 String vpnName = getExtNetworkVpnName(routerId);
 333                 if (vpnName != null) {
 334                     //List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 335                     //if (externalIps != null) {
 336                     for (String externalIp : externalIpsCache) {
 337                         externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 338                     }
 339                 } else {
 340                     LOG.debug("vpn is not associated to extn/w for router {}", routerName);
 341                 }
 342             } else {
 343                 LOG.debug("No ExternalIps found for subnets under router {}, no bgp routes need to be cleared",routerName);
 344             }
 345             return true;
 346         }
 347         //checking elected switch health status
 348         if (!getSwitchStatus(naptSwitch)) {
 349             LOG.error("Newly elected Napt switch {} for router {} is down", naptSwitch, routerName);
 350             return true;
 351         }
 352         LOG.debug("New NaptSwitch {} is up for Router {} and can proceed for flow installation",naptSwitch, routerName);
 353         //update napt model for new napt switch
 354         boolean naptUpdated = updateNaptSwitch(routerName, naptSwitch);
 355         if (naptUpdated) {
 356             //update group of ordinary switch point to naptSwitch tunnel port
 357             updateNaptSwitchBucketStatus(routerName, naptSwitch);
 358         } else {
 359             LOG.error("Failed to update naptSwitch model for newNaptSwitch {} for router {}",naptSwitch, routerName);
 360         }
 361
 362         //update table26 forward packets to table46(outbound napt table)
 363         FlowEntity flowEntity = buildSnatFlowEntityForNaptSwitch(naptSwitch, routerName, routerVpnId, NatConstants.ADD_FLOW);
 364         if (flowEntity == null) {
 365             LOG.debug("Failed to populate flowentity for router {} in naptSwitch {}", routerName, naptSwitch);
 366         } else {
 367             LOG.debug("Successfully installed flow in naptSwitch {} for router {}", naptSwitch, routerName);
 368             mdsalManager.installFlow(flowEntity);
 369         }
 370
 371         installSnatFlows(routerName,routerId,naptSwitch,routerVpnId);
 372
 373         boolean flowInstalledStatus = handleNatFlowsInNewNaptSwitch(routerId, dpnId, naptSwitch,routerVpnId);
 374         if (flowInstalledStatus) {
 375             LOG.debug("Installed all active session flows in newNaptSwitch {} for routerName {}", naptSwitch, routerName);
 376         } else {
 377             LOG.error("Failed to install flows in newNaptSwitch {} for routerId {}", naptSwitch, routerId);
 378         }
 379
 380         //remove group in new naptswitch, coz this switch acted previously as ordinary switch
 381         long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 382         GroupEntity groupEntity = null;
 383         try {
 384             groupEntity = MDSALUtil.buildGroupEntity(naptSwitch, groupId, routerName,
 385                     GroupTypes.GroupAll, null);
 386             LOG.info("NAT Service : Removing NAPT Group in new naptSwitch {}", naptSwitch);
 387             mdsalManager.removeGroup(groupEntity);
 388         } catch (Exception ex) {
 389             LOG.debug("NAT Service : Failed to remove group in new naptSwitch {} : {}",groupEntity,ex);
 390         }
 391         return true;
 392     }
 393
 394     private String getExtNetworkVpnName(long routerId) {
 395         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 396         if(networkId == null) {
 397             LOG.error("networkId is null for the router ID {}", routerId);
 398         } else {
 399             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId, LOG);
 400             if (vpnName != null) {
 401                 LOG.debug("retrieved vpn name {} associated with ext nw {} in router {}",
 402                         vpnName,networkId,routerId);
 403                 return vpnName;
 404             } else {
 405                 LOG.error("No VPN associated with ext nw {} belonging to routerId {}",
 406                         networkId, routerId);
 407             }
 408         }
 409         return null;
 410     }
 411
 412     public void updateNaptSwitchBucketStatus(String routerName, BigInteger naptSwitch) {
 413         LOG.debug("updateNaptSwitchBucketStatus method is called");
 414
 415         List<BigInteger> dpnList = naptSwitchSelector.getDpnsForVpn(routerName);
 416         //List<BigInteger> dpnList = getDpnListForRouter(routerName);
 417         if (dpnList == null || dpnList.isEmpty()) {
 418             LOG.debug("No switches found for router {}",routerName);
 419             return;
 420         }
 421         for (BigInteger dpn : dpnList) {
 422             if (!dpn.equals(naptSwitch)) {
 423                 LOG.debug("Updating SNAT_TABLE missentry for DpnId {} which is not naptSwitch for router {}",dpn,routerName);
 424                 List<BucketInfo> bucketInfoList = handleGroupInNeighborSwitches(dpn, routerName, naptSwitch);
 425                 if (bucketInfoList == null) {
 426                     LOG.debug("Failed to populate bucketInfo for orinaryswitch {} whose naptSwitch {} for router {} ",
 427                             dpn,naptSwitch,routerName);
 428                     return;
 429                 }
 430                 modifySnatGroupEntry(dpn, bucketInfoList, routerName);
 431             }
 432         }
 433     }
 434
 435     private boolean handleNatFlowsInNewNaptSwitch(Long routerId,BigInteger oldNaptSwitch, BigInteger newNaptSwitch,Long routerVpnId) {
 436         LOG.debug("Proceeding to install flows in newNaptSwitch {} for routerId {}", newNaptSwitch,routerId);
 437         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker,routerId);
 438         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 439             LOG.debug("No Internal Ip Port mapping associated to router {}, no flows need to be installed in" +
 440                     "newNaptSwitch {}", routerId, newNaptSwitch);
 441             return true;
 442         }
 443         //getvpnId
 444         Long vpnId = getVpnIdForRouter(routerId);
 445         if (vpnId == NatConstants.INVALID_ID) {
 446             LOG.error("Invalid vpnId for routerId {}",routerId);
 447             return false;
 448         }
 449         Long bgpVpnId;
 450         if(routerId.equals(routerVpnId)) {
 451             bgpVpnId = NatConstants.INVALID_ID;
 452         } else {
 453             bgpVpnId = routerVpnId;
 454         }
 455         LOG.debug("retrieved bgpVpnId {} for router {}",bgpVpnId,routerId);
 456         for (IntextIpProtocolType protocolType : ipPortMapping.getIntextIpProtocolType()) {
 457             if (protocolType.getIpPortMap() == null || protocolType.getIpPortMap().isEmpty()) {
 458                 LOG.debug("No {} session associated to router {}", protocolType.getProtocol(), routerId);
 459                 return true;
 460             }
 461             for (IpPortMap intIpPortMap : protocolType.getIpPortMap()) {
 462                 String internalIpAddress = intIpPortMap.getIpPortInternal().split(":")[0];
 463                 String intportnum = intIpPortMap.getIpPortInternal().split(":")[1];
 464
 465                 //Get the external IP address and the port from the model
 466                 NAPTEntryEvent.Protocol proto = protocolType.getProtocol().toString().equals(ProtocolTypes.TCP.toString())
 467                         ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP;
 468                 IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
 469                         internalIpAddress, intportnum, proto);
 470                 if (ipPortExternal == null) {
 471                     LOG.debug("External Ipport mapping is not found for internalIp {} with port {}", internalIpAddress, intportnum);
 472                     continue;
 473                 }
 474                 String externalIpAddress = ipPortExternal.getIpAddress();
 475                 Integer extportNumber = ipPortExternal.getPortNum();
 476                 LOG.debug("ExternalIPport {}:{} mapping for internal ipport {}:{}",externalIpAddress,extportNumber,
 477                         internalIpAddress,intportnum);
 478
 479                 SessionAddress sourceAddress = new SessionAddress(internalIpAddress,Integer.valueOf(intportnum));
 480                 SessionAddress externalAddress = new SessionAddress(externalIpAddress,extportNumber);
 481
 482                 //checking naptSwitch status before installing flows
 483                 if(getSwitchStatus(newNaptSwitch)) {
 484                     //Install the flow in newNaptSwitch Outbound NAPT table.
 485                     try {
 486                         NaptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 487                                 vpnId,  routerId, bgpVpnId, sourceAddress, externalAddress, proto);
 488                     } catch (Exception ex) {
 489                         LOG.error("Failed to add flow in OUTBOUND_NAPT_TABLE for routerid {} dpnId {} ipport {}:{} proto {}" +
 490                                 "extIpport {}:{} BgpVpnId {} - {}", routerId, newNaptSwitch, internalIpAddress
 491                                 , intportnum, proto, externalAddress, extportNumber,bgpVpnId,ex);
 492                         return false;
 493                     }
 494                     LOG.debug("Successfully installed a flow in SecondarySwitch {} Outbound NAPT table for router {} " +
 495                             "ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}", newNaptSwitch,routerId, internalIpAddress
 496                             , intportnum, proto, externalAddress, extportNumber,bgpVpnId);
 497                     //Install the flow in newNaptSwitch Inbound NAPT table.
 498                     try {
 499                         NaptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 500                                 vpnId, routerId, bgpVpnId, externalAddress, sourceAddress, proto);
 501                     } catch (Exception ex) {
 502                         LOG.error("Failed to add flow in INBOUND_NAPT_TABLE for routerid {} dpnId {} extIpport{}:{} proto {} " +
 503                                         "ipport {}:{} BgpVpnId {}", routerId, newNaptSwitch, externalAddress, extportNumber, proto,
 504                                 internalIpAddress, intportnum,bgpVpnId);
 505                         return false;
 506                     }
 507                     LOG.debug("Successfully installed a flow in SecondarySwitch {} Inbound NAPT table for router {} " +
 508                             "ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}", newNaptSwitch,routerId, internalIpAddress
 509                             , intportnum, proto, externalAddress, extportNumber,bgpVpnId);
 510
 511                 } else {
 512                     LOG.error("NewNaptSwitch {} gone down while installing flows from oldNaptswitch {}",
 513                             newNaptSwitch,oldNaptSwitch);
 514                     return false;
 515                 }
 516             }
 517         }
 518         return true;
 519     }
 520
 521     private Long getVpnIdForRouter(Long routerId) {
 522         try {
 523             //getvpnId
 524             Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 525             if (networkId == null) {
 526                 LOG.debug("network is not associated to router {}", routerId);
 527             } else {
 528                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 529                 if (vpnUuid == null) {
 530                     LOG.debug("vpn is not associated for network {} in router {}", networkId, routerId);
 531                 } else {
 532                     Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 533                     if (vpnId > 0) {
 534                         LOG.debug("retrieved vpnId {} for router {}",vpnId,routerId);
 535                         return vpnId;
 536                     } else {
 537                         LOG.debug("retrieved invalid vpn Id");
 538                     }
 539                 }
 540             }
 541         } catch (Exception ex){
 542             LOG.debug("Exception while retrieving vpnId for router {} - {}", routerId, ex);
 543         }
 544         return NatConstants.INVALID_ID;
 545     }
 546
 547     public boolean getSwitchStatus(BigInteger switchId){
 548         NodeId nodeId = new NodeId("openflow:" + switchId);
 549         LOG.debug("Querying switch with dpnId {} is up/down", nodeId);
 550         InstanceIdentifier<Node> nodeInstanceId = InstanceIdentifier.builder(Nodes.class)
 551                 .child(Node.class, new NodeKey(nodeId)).build();
 552         Optional<Node> nodeOptional = NatUtil.read(dataBroker,LogicalDatastoreType.OPERATIONAL,nodeInstanceId);
 553         if (nodeOptional.isPresent()) {
 554             LOG.debug("Switch {} is up", nodeId);
 555             return true;
 556         }
 557         LOG.debug("Switch {} is down", nodeId);
 558         return false;
 559     }
 560
 561     public List<BucketInfo> handleGroupInPrimarySwitch() {
 562         List<BucketInfo> listBucketInfo = new ArrayList<>();
 563         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 564         listActionInfoPrimary.add(new ActionInfo(ActionType.nx_resubmit,
 565                 new String[]{String.valueOf(NwConstants.INTERNAL_TUNNEL_TABLE)}));
 566         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 567         listBucketInfo.add(bucketPrimary);
 568         return listBucketInfo;
 569     }
 570
 571     public List<BucketInfo> handleGroupInNeighborSwitches(BigInteger dpnId, String routerName, BigInteger naptSwitch) {
 572         List<BucketInfo> listBucketInfo = new ArrayList<>();
 573         String ifNamePrimary;
 574         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 575         if (routerId == NatConstants.INVALID_ID) {
 576             LOG.error("Invalid routerId returned for routerName {}",routerName);
 577             return listBucketInfo;
 578         }
 579         ifNamePrimary = getTunnelInterfaceName(dpnId, naptSwitch);
 580         if (ifNamePrimary != null) {
 581             LOG.debug("TunnelInterface {} between ordinary switch {} and naptSwitch {}",ifNamePrimary,dpnId,naptSwitch);
 582             List<ActionInfo> listActionInfoPrimary = NatUtil.getEgressActionsForInterface(interfaceManager, ifNamePrimary, routerId);
 583             BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 584             listBucketInfo.add(bucketPrimary);
 585         } else {
 586             LOG.debug("No TunnelInterface between ordinary switch {} and naptSwitch {}",dpnId,naptSwitch);
 587         }
 588         return listBucketInfo;
 589     }
 590
 591     protected void installSnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 592         GroupEntity groupEntity = null;
 593         try {
 594             long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 595             LOG.debug("install SnatMissEntry for groupId {} for dpnId {} for router {}", groupId, dpnId,routerName);
 596             groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, routerName,
 597                     GroupTypes.GroupAll, bucketInfo);
 598             mdsalManager.installGroup(groupEntity);
 599             LOG.debug("installed the SNAT to NAPT GroupEntity:{}", groupEntity);
 600         } catch (Exception ex) {
 601             LOG.error("Failed to install group for groupEntity {} : {}",groupEntity,ex);
 602         }
 603     }
 604
 605     private void modifySnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 606         installSnatGroupEntry(dpnId,bucketInfo,routerName);
 607         LOG.debug("modified SnatMissEntry for dpnId {} of router {}",dpnId,routerName);
 608     }
 609
 610     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 611         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 612         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 613
 614         try {
 615             Future<RpcResult<GetTunnelInterfaceNameOutput>> result = itmManager.getTunnelInterfaceName(
 616                     new GetTunnelInterfaceNameInputBuilder().setSourceDpid(srcDpId).setDestinationDpid(dstDpId)
 617                             .setTunnelType(tunType).build());
 618             rpcResult = result.get();
 619             if(!rpcResult.isSuccessful()) {
 620                 tunType = TunnelTypeGre.class;
 621                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 622                         .setSourceDpid(srcDpId)
 623                         .setDestinationDpid(dstDpId)
 624                         .setTunnelType(tunType)
 625                         .build());
 626                 rpcResult = result.get();
 627                 if(!rpcResult.isSuccessful()) {
 628                     LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 629                 } else {
 630                     return rpcResult.getResult().getInterfaceName();
 631                 }
 632                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 633             } else {
 634                 return rpcResult.getResult().getInterfaceName();
 635             }
 636         } catch (InterruptedException | ExecutionException e) {
 637             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {} : {}",
 638                     srcDpId, dstDpId, e);
 639         }
 640
 641         return null;
 642     }
 643
 644     public boolean updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
 645         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().setKey(new RouterToNaptSwitchKey(routerName))
 646                 .setPrimarySwitchId(naptSwitchId).build();
 647         try {
 648             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 649                     NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
 650         } catch (Exception ex) {
 651             LOG.error("Failed to write naptSwitch {} for router {} in ds",
 652                     naptSwitchId,routerName);
 653             return false;
 654         }
 655         LOG.debug("Successfully updated naptSwitch {} for router {} in ds",
 656                 naptSwitchId,routerName);
 657         return true;
 658     }
 659
 660     public FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long groupId, long routerVpnId, int addordel) {
 661
 662         FlowEntity flowEntity;
 663         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 664         matches.add(new MatchInfo(MatchFieldType.eth_type,
 665                 new long[]{ 0x0800L }));
 666         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 667                 MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID }));
 668
 669         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 670
 671         if (addordel == NatConstants.ADD_FLOW) {
 672             List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 673             List<ActionInfo> actionsInfo = new ArrayList<ActionInfo>();
 674
 675             ActionInfo actionSetField = new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] {
 676                     BigInteger.valueOf(routerVpnId)}) ;
 677             actionsInfo.add(actionSetField);
 678             LOG.debug("Setting the tunnel to the list of action infos {}", actionsInfo);
 679             actionsInfo.add(new ActionInfo(ActionType.group, new String[] {String.valueOf(groupId)}));
 680             instructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfo));
 681
 682             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 683                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 684                     NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 685         } else {
 686             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 687                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 688                     NwConstants.COOKIE_SNAT_TABLE, matches, null);
 689         }
 690         return flowEntity;
 691     }
 692
 693     public FlowEntity buildSnatFlowEntityForNaptSwitch(BigInteger dpId, String routerName, long routerVpnId, int addordel) {
 694
 695         FlowEntity flowEntity;
 696         List<MatchInfo> matches = new ArrayList<>();
 697         matches.add(new MatchInfo(MatchFieldType.eth_type,
 698                 new long[]{ 0x0800L }));
 699         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 700                 MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID }));
 701
 702         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 703
 704         if (addordel == NatConstants.ADD_FLOW) {
 705             List<InstructionInfo> instructions = new ArrayList<>();
 706
 707             instructions.add(new InstructionInfo(InstructionType.goto_table, new long[]
 708                     { NwConstants.OUTBOUND_NAPT_TABLE }));
 709
 710             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 711                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 712                     NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 713         } else {
 714             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 715                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 716                     NwConstants.COOKIE_SNAT_TABLE, matches, null);
 717         }
 718         return flowEntity;
 719     }
 720
 721     private String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 722         return new StringBuilder().append(NatConstants.SNAT_FLOWID_PREFIX).append(dpnId).append(NatConstants.FLOWID_SEPARATOR).
 723                 append(tableId).append(NatConstants.FLOWID_SEPARATOR).append(routerID).toString();
 724     }
 725
 726     protected void installSnatFlows(String routerName,Long routerId,BigInteger naptSwitch,Long routerVpnId) {
 727
 728         if(routerId.equals(routerVpnId)) {
 729             LOG.debug("Installing flows for router with internalvpnId");
 730             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 731             LOG.debug("installTerminatingServiceTblEntry in naptswitch with dpnId {} for routerName {} with routerId {}",
 732                     naptSwitch, routerName,routerId);
 733             externalRouterListener.installTerminatingServiceTblEntry(naptSwitch, routerName);
 734
 735             //Install default flows punting to controller in table 46(OutBoundNapt table)
 736             LOG.debug("installOutboundMissEntry in naptswitch with dpnId {} for routerName {} with routerId {}",
 737                     naptSwitch, routerName, routerId);
 738             externalRouterListener.createOutboundTblEntry(naptSwitch, routerId);
 739
 740             //Table 47 point to table 21 for inbound traffic
 741             LOG.debug("installNaptPfibEntry in naptswitch with dpnId {} for router {}", naptSwitch, routerId);
 742             externalRouterListener.installNaptPfibEntry(naptSwitch, routerId);
 743         } else {
 744             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 745             LOG.debug("installTerminatingServiceTblEntry in naptswitch with dpnId {} for routerName {} with BgpVpnId {}",
 746                     naptSwitch, routerName, routerVpnId);
 747             externalRouterListener.installTerminatingServiceTblEntryWithUpdatedVpnId(naptSwitch, routerName, routerVpnId);
 748
 749             //Install default flows punting to controller in table 46(OutBoundNapt table)
 750             LOG.debug("installOutboundMissEntry in naptswitch with dpnId {} for routerName {} with BgpVpnId {}",
 751                     naptSwitch, routerName, routerVpnId);
 752             externalRouterListener.createOutboundTblEntryWithBgpVpn(naptSwitch, routerId, routerVpnId);
 753
 754             //Table 47 point to table 21 for inbound traffic
 755             LOG.debug("installNaptPfibEntry in naptswitch with dpnId {} for router {} with BgpVpnId {}",
 756                     naptSwitch, routerId, routerVpnId);
 757             externalRouterListener.installNaptPfibEntryWithBgpVpn(naptSwitch, routerId, routerVpnId);
 758         }
 759
 760         String vpnName = getExtNetworkVpnName(routerId);
 761         if(vpnName != null) {
 762             //Table 47 point to table 21 for outbound traffic
 763             long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 764             if(vpnId > 0) {
 765                 LOG.debug("installNaptPfibEntry fin naptswitch with dpnId {} for BgpVpnId {}", naptSwitch, vpnId);
 766                 externalRouterListener.installNaptPfibEntry(naptSwitch, vpnId);
 767             } else {
 768                 LOG.debug("Associated BgpvpnId not found for router {}",routerId);
 769             }
 770
 771             //Install Fib entries for ExternalIps & program 36 -> 44
 772             List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 773             if (externalIps != null) {
 774                 for (String externalIp : externalIps) {
 775                     LOG.debug("advToBgpAndInstallFibAndTsFlows in naptswitch id {} with vpnName {} and externalIp {}",
 776                             naptSwitch, vpnName, externalIp);
 777                     externalRouterListener.advToBgpAndInstallFibAndTsFlows(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 778                             vpnName, routerId, externalIp, vpnService, fibService, bgpManager, dataBroker, LOG);
 779                     LOG.debug("Successfully added fib entries in naptswitch {} for router {} with external IP {}", naptSwitch,
 780                             routerId, externalIp);
 781                 }
 782             } else {
 783                 LOG.debug("External Ip not found for routerId {}",routerId);
 784             }
 785         } else {
 786             LOG.debug("Associated vpnName not found for router {}",routerId);
 787         }
 788     }
 789
 790     protected void bestEffortDeletion(long routerId,String routerName,HashMap<String,Long> externalIpLabel) {
 791         List<String> newExternalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 792         if (newExternalIps != null && externalIpsCache != null) {
 793             Set<String> originalSubnetIds = Sets.newHashSet(externalIpsCache);
 794             Set<String> updatedSubnetIds = Sets.newHashSet(newExternalIps);
 795             Sets.SetView<String> removeExternalIp = Sets.difference(originalSubnetIds, updatedSubnetIds);
 796             if (removeExternalIp.isEmpty()) {
 797                 LOG.debug("No external Ip needed to be removed in bestEffortDeletion method for router {}",routerName);
 798                 return;
 799             }
 800             String vpnName = getExtNetworkVpnName(routerId);
 801             if (vpnName == null) {
 802                 LOG.debug("Vpn is not associated to externalN/w of router {}",routerName);
 803                 return;
 804             }
 805             if (externalIpLabel == null || externalIpLabel.size() == 0) {
 806                 LOG.debug("ExternalIpLabel map is empty for router {}",routerName);
 807                 return;
 808             }
 809             BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
 810             if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 811                 LOG.debug("No naptSwitch is selected for router {}", routerName);
 812                 return;
 813             }
 814             Long label;
 815             for (String externalIp : removeExternalIp) {
 816                 if (externalIpLabel.containsKey(externalIp)) {
 817                     label = externalIpLabel.get(externalIp);
 818                     LOG.debug("Label {} for ExternalIp {} for router {}",label,externalIp,routerName);
 819                 } else {
 820                     LOG.debug("Label for ExternalIp {} is not found for router {}",externalIp,routerName);
 821                     continue;
 822                 }
 823                 externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 824                 externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerId, externalIp, vpnName,label);
 825                 LOG.debug("Successfully removed fib entries in switch {} for router {} and externalIps {}",
 826                             naptSwitch, routerId, externalIp);
 827             }
 828         } else {
 829             LOG.debug("No external IP found for router {}",routerId);
 830         }
 831     }
 832 }