git.opendaylight Code Review - netvirt.git/blob

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.collect.Sets;
  12
  13 import org.opendaylight.genius.mdsalutil.ActionInfo;
  14 import org.opendaylight.genius.mdsalutil.ActionType;
  15 import org.opendaylight.genius.mdsalutil.BucketInfo;
  16 import org.opendaylight.genius.mdsalutil.FlowEntity;
  17 import org.opendaylight.genius.mdsalutil.GroupEntity;
  18 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  19 import org.opendaylight.genius.mdsalutil.InstructionType;
  20 import org.opendaylight.genius.mdsalutil.MatchFieldType;
  21 import org.opendaylight.genius.mdsalutil.MatchInfo;
  22 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  23 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  24 import org.opendaylight.genius.mdsalutil.NwConstants;
  25 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeId;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  56 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  57 import org.opendaylight.yangtools.yang.common.RpcResult;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 import java.math.BigInteger;
  62 import java.util.ArrayList;
  63 import java.util.List;
  64 import java.util.Set;
  65 import java.util.concurrent.ExecutionException;
  66 import java.util.concurrent.Future;
  67 import java.util.HashMap;
  68
  69 public class NaptSwitchHA {
  70     private static final Logger LOG = LoggerFactory.getLogger(NaptSwitchHA.class);
  71     private final DataBroker dataBroker;
  72     private final IMdsalApiManager mdsalManager;
  73     private final ItmRpcService itmManager;
  74     private final OdlInterfaceRpcService interfaceManager;
  75     private final IdManagerService idManager;
  76     private final NAPTSwitchSelector naptSwitchSelector;
  77     private final ExternalRoutersListener externalRouterListener;
  78     private final IBgpManager bgpManager;
  79     private final VpnRpcService vpnService;
  80     private final FibRpcService fibService;
  81     private List<String> externalIpsCache;
  82     private HashMap<String,Long> externalIpsLabel;
  83
  84     public NaptSwitchHA(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  85                         final ExternalRoutersListener externalRouterListener,
  86                         final ItmRpcService itmManager,
  87                         final OdlInterfaceRpcService interfaceManager,
  88                         final IdManagerService idManager,
  89                         final NAPTSwitchSelector naptSwitchSelector,
  90                         final IBgpManager bgpManager,
  91                         final VpnRpcService vpnService,
  92                         final FibRpcService fibService) {
  93         this.dataBroker = dataBroker;
  94         this.mdsalManager = mdsalManager;
  95         this.externalRouterListener = externalRouterListener;
  96         this.itmManager = itmManager;
  97         this.interfaceManager = interfaceManager;
  98         this.idManager = idManager;
  99         this.naptSwitchSelector = naptSwitchSelector;
 100         this.bgpManager = bgpManager;
 101         this.vpnService = vpnService;
 102         this.fibService =fibService;
 103     }
 104
 105     /* This method checks the switch that gone down is a NaptSwitch for a router.
 106        If it is a NaptSwitch
 107           1) selects new NAPT switch
 108           2) installs nat flows in new NAPT switch
 109           table 21(FIB)->26(PSNAT)->group(resubmit/napttunnel)->36(Terminating)->46(outbound)->47(resubmit)->21
 110           3) modify the group and miss entry flow in other vSwitches pointing to newNaptSwitch
 111           4) Remove nat flows in oldNaptSwitch
 112      */
 113     /*public void handleNaptSwitchDown(BigInteger dpnId){
 114
 115         LOG.debug("handleNaptSwitchDown method is called with dpnId {}",dpnId);
 116         BigInteger naptSwitch;
 117         try {
 118             NaptSwitches naptSwitches = NatUtil.getNaptSwitch(dataBroker);
 119             if (naptSwitches == null || naptSwitches.getRouterToNaptSwitch() == null || naptSwitches.getRouterToNaptSwitch().isEmpty()) {
 120                 LOG.debug("NaptSwitchDown: NaptSwitch is not allocated for none of the routers");
 121                 return;
 122             }
 123             for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.getRouterToNaptSwitch()) {
 124                 String routerName = routerToNaptSwitch.getRouterName();
 125                 naptSwitch = routerToNaptSwitch.getPrimarySwitchId();
 126                 boolean naptStatus = isNaptSwitchDown(routerName,dpnId,naptSwitch);
 127                 if (!naptStatus) {
 128                     LOG.debug("NaptSwitchDown: Switch with DpnId {} is not naptSwitch for router {}",
 129                             dpnId, routerName);
 130                 } else {
 131                     removeSnatFlowsInOldNaptSwitch(routerName,naptSwitch);
 132                     return;
 133                 }
 134             }
 135         } catch (Exception ex) {
 136             LOG.error("Exception in handleNaptSwitchDown method {}",ex);
 137         }
 138     }*/
 139
 140     protected void removeSnatFlowsInOldNaptSwitch(String routerName, BigInteger naptSwitch,HashMap<String,Long> externalIpmap) {
 141         externalIpsLabel = externalIpmap;
 142         //remove SNAT flows in old NAPT SWITCH
 143         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 144         if (routerId == NatConstants.INVALID_ID) {
 145             LOG.error("Invalid routerId returned for routerName {}",routerName);
 146             return;
 147         }
 148
 149         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table
 150         String tsFlowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
 151         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
 152
 153         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}"
 154                 ,NwConstants.INTERNAL_TUNNEL_TABLE, naptSwitch, routerId);
 155         mdsalManager.removeFlow(tsNatFlowEntity);
 156
 157         //Remove the Outbound flow entry which forwards the packet to Outbound NAPT Table
 158         String outboundNatFlowRef = externalRouterListener.getFlowRefOutbound(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
 159         FlowEntity outboundNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch,
 160                 NwConstants.OUTBOUND_NAPT_TABLE, outboundNatFlowRef);
 161         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}"
 162                 ,NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId);
 163         mdsalManager.removeFlow(outboundNatFlowEntity);
 164
 165         //Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for inbound traffic matching on the router ID.
 166         String naptPFibflowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, routerId);
 167         FlowEntity naptPFibFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE, naptPFibflowRef);
 168         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}",
 169                 NwConstants.NAPT_PFIB_TABLE, naptSwitch, routerId);
 170         mdsalManager.removeFlow(naptPFibFlowEntity);
 171
 172         //Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for outbound traffic matching on the vpn ID.
 173         boolean switchSharedByRouters = false;
 174         Uuid extNetworkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 175         if (extNetworkId != null) {
 176             List<String> routerNamesAssociated = getRouterIdsForExtNetwork(extNetworkId);
 177             if (routerNamesAssociated != null) {
 178                 for (String routerNameAssociated : routerNamesAssociated) {
 179                     if (!routerNameAssociated.equals(routerName)) {
 180                         Long routerIdAssociated = NatUtil.getVpnId(dataBroker,routerNameAssociated);
 181                         BigInteger naptDpn = NatUtil.getPrimaryNaptfromRouterId(dataBroker,routerIdAssociated);
 182                         if (naptDpn != null && naptDpn.equals(naptSwitch)) {
 183                             LOG.debug("Napt switch {} is also acting as primary for router {}",routerIdAssociated);
 184                             switchSharedByRouters = true;
 185                             break;
 186                         }
 187                     }
 188                 }
 189                 if (!switchSharedByRouters) {
 190                     Long vpnId = getVpnIdForRouter(routerId);
 191                     if (vpnId != NatConstants.INVALID_ID) {
 192                         String naptFibflowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, vpnId);
 193                         FlowEntity naptFibFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE,naptFibflowRef);
 194                         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and vpnId {}",
 195                                 NwConstants.NAPT_PFIB_TABLE, naptSwitch, vpnId);
 196                         mdsalManager.removeFlow(naptFibFlowEntity);
 197                     } else {
 198                         LOG.error("Invalid vpnId retrieved for routerId {}",routerId);
 199                         return;
 200                     }
 201                 }
 202             }
 203         }
 204
 205         //Remove Fib entries,tables 20->44 ,36-> 44
 206         String vpnName = getExtNetworkVpnName(routerId);
 207         if (vpnName == null) {
 208             LOG.debug("Vpn is not associated to externalN/w of router {}",routerName);
 209         } else {
 210             if (externalIpsLabel != null) {
 211                 for (String externalIp : externalIpsLabel.keySet()) {
 212                     Long label = externalIpsLabel.get(externalIp);
 213                     externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerId, externalIp, vpnName,label);
 214                     LOG.debug("Successfully removed fib entries in old naptswitch {} for router {} and externalIps {} label {}",
 215                             naptSwitch, routerId,externalIp,label);
 216                 }
 217             } else {
 218                 List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 219                 if (externalIps != null) {
 220                     Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 221                     if (networkId != null) {
 222                         externalRouterListener.clearFibTsAndReverseTraffic(naptSwitch, routerId, networkId, externalIps, null);
 223                         LOG.debug("Successfully removed fib entries in old naptswitch {} for router {} with networkId {} and externalIps {}",
 224                                     naptSwitch,routerId,networkId,externalIps);
 225                     } else {
 226                         LOG.debug("External network not associated to router {}", routerId);
 227                     }
 228                 } else {
 229                     LOG.debug("ExternalIps not found for router {}",routerName);
 230                 }
 231             }
 232         }
 233
 234         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
 235         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
 236         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 237             LOG.debug("No Internal Ip Port mapping associated to router {}, no flows need to be removed in" +
 238                     "oldNaptSwitch {}", routerId, naptSwitch);
 239             return;
 240         }
 241         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
 242         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
 243         for(IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
 244             if (intextIpProtocolType.getIpPortMap() == null || intextIpProtocolType.getIpPortMap().isEmpty()) {
 245                 LOG.debug("No {} session associated to router {},no flows need to be removed in oldNaptSwitch {}",
 246                         intextIpProtocolType.getProtocol(),routerId,naptSwitch);
 247                 break;
 248             }
 249             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
 250             for(IpPortMap ipPortMap : ipPortMaps) {
 251                 String ipPortInternal = ipPortMap.getIpPortInternal();
 252                 String[] ipPortParts = ipPortInternal.split(":");
 253                 if(ipPortParts.length != 2) {
 254                     LOG.error("Unable to retrieve the Internal IP and port");
 255                     continue;
 256                 }
 257                 String internalIp = ipPortParts[0];
 258                 String internalPort = ipPortParts[1];
 259
 260                 //Build and remove flow in outbound NAPT table
 261                 String switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, String.valueOf(routerId),
 262                         internalIp, Integer.valueOf(internalPort));
 263                 FlowEntity outboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 264                         cookieSnatFlow, switchFlowRef);
 265
 266                 LOG.info("Remove the flow in table {} for old napt switch with the DPN ID {} and router ID {}",
 267                         NwConstants.OUTBOUND_NAPT_TABLE,naptSwitch, routerId);
 268                 mdsalManager.removeFlow(outboundNaptFlowEntity);
 269
 270                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
 271                 if (ipPortExternal == null) {
 272                     LOG.debug("External Ipport mapping not found for internalIp {} with port {} for router", internalIp,
 273                             internalPort, routerId);
 274                     continue;
 275                 }
 276                 String externalIp = ipPortExternal.getIpAddress();
 277                 int externalPort = ipPortExternal.getPortNum();
 278
 279                 //Build and remove flow in  inbound NAPT table
 280                 switchFlowRef = NatUtil.getNaptFlowRef(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, String.valueOf(routerId),
 281                         externalIp, externalPort);
 282                 FlowEntity inboundNaptFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 283                         cookieSnatFlow, switchFlowRef);
 284
 285                 LOG.info("Remove the flow in table {} for old napt switch with the DPN ID {} and router ID {}",
 286                         NwConstants.INBOUND_NAPT_TABLE,naptSwitch, routerId);
 287                 mdsalManager.removeFlow(inboundNaptFlowEntity);
 288             }
 289         }
 290
 291     }
 292
 293     private List<String> getRouterIdsForExtNetwork(Uuid extNetworkId) {
 294         List<String> routerUuidsAsString = new ArrayList<>();
 295         InstanceIdentifier<Networks> extNetwork = InstanceIdentifier.builder(ExternalNetworks.class).child
 296                 (Networks.class, new NetworksKey(extNetworkId)).build();
 297         Optional<Networks> extNetworkData = NatUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, extNetwork);
 298         if (extNetworkData.isPresent()) {
 299             List<Uuid> routerUuids= extNetworkData.get().getRouterIds();
 300             if (routerUuids != null){
 301                 for(Uuid routerUuid : routerUuids){
 302                     routerUuidsAsString.add(routerUuid.getValue());
 303                 }
 304             }
 305         }
 306         return routerUuidsAsString;
 307     }
 308
 309     public boolean isNaptSwitchDown(String routerName, BigInteger dpnId , BigInteger naptSwitch,Long routerVpnId,List<String> externalIpCache) {
 310         externalIpsCache = externalIpCache;
 311         if (!naptSwitch.equals(dpnId)) {
 312             LOG.debug("DpnId {} is not a naptSwitch {} for Router {}",dpnId, naptSwitch, routerName);
 313             return false;
 314         }
 315         LOG.debug("NaptSwitch {} is down for Router {}", naptSwitch, routerName);
 316         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 317         if (routerId == NatConstants.INVALID_ID) {
 318             LOG.error("Invalid routerId returned for routerName {}", routerName);
 319             return true;
 320         }
 321         //elect a new NaptSwitch
 322         naptSwitch = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 323         if (naptSwitch.equals(BigInteger.ZERO)) {
 324             LOG.info("No napt switch is elected since all the switches for router {} are down",routerName);
 325             boolean naptUpdatedStatus = updateNaptSwitch(routerName,naptSwitch);
 326             if(!naptUpdatedStatus) {
 327                 LOG.debug("Failed to update naptSwitch {} for router {} in ds", naptSwitch,routerName);
 328             }
 329             //clearBgpRoutes
 330             if (externalIpsCache != null) {
 331                 String vpnName = getExtNetworkVpnName(routerId);
 332                 if (vpnName != null) {
 333                     //List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 334                     //if (externalIps != null) {
 335                     for (String externalIp : externalIpsCache) {
 336                         externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 337                     }
 338                 } else {
 339                     LOG.debug("vpn is not associated to extn/w for router {}", routerName);
 340                 }
 341             } else {
 342                 LOG.debug("No ExternalIps found for subnets under router {}, no bgp routes need to be cleared",routerName);
 343             }
 344             return true;
 345         }
 346         //checking elected switch health status
 347         if (!getSwitchStatus(naptSwitch)) {
 348             LOG.error("Newly elected Napt switch {} for router {} is down", naptSwitch, routerName);
 349             return true;
 350         }
 351         LOG.debug("New NaptSwitch {} is up for Router {} and can proceed for flow installation",naptSwitch, routerName);
 352         //update napt model for new napt switch
 353         boolean naptUpdated = updateNaptSwitch(routerName, naptSwitch);
 354         if (naptUpdated) {
 355             //update group of ordinary switch point to naptSwitch tunnel port
 356             updateNaptSwitchBucketStatus(routerName, naptSwitch);
 357         } else {
 358             LOG.error("Failed to update naptSwitch model for newNaptSwitch {} for router {}",naptSwitch, routerName);
 359         }
 360
 361         //update table26 forward packets to table46(outbound napt table)
 362         FlowEntity flowEntity = buildSnatFlowEntityForNaptSwitch(naptSwitch, routerName, routerVpnId, NatConstants.ADD_FLOW);
 363         if (flowEntity == null) {
 364             LOG.debug("Failed to populate flowentity for router {} in naptSwitch {}", routerName, naptSwitch);
 365         } else {
 366             LOG.debug("Successfully installed flow in naptSwitch {} for router {}", naptSwitch, routerName);
 367             mdsalManager.installFlow(flowEntity);
 368         }
 369
 370         installSnatFlows(routerName,routerId,naptSwitch,routerVpnId);
 371
 372         boolean flowInstalledStatus = handleNatFlowsInNewNaptSwitch(routerId, dpnId, naptSwitch,routerVpnId);
 373         if (flowInstalledStatus) {
 374             LOG.debug("Installed all active session flows in newNaptSwitch {} for routerName {}", naptSwitch, routerName);
 375         } else {
 376             LOG.error("Failed to install flows in newNaptSwitch {} for routerId {}", naptSwitch, routerId);
 377         }
 378
 379         //remove group in new naptswitch, coz this switch acted previously as ordinary switch
 380         long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 381         GroupEntity groupEntity = null;
 382         try {
 383             groupEntity = MDSALUtil.buildGroupEntity(naptSwitch, groupId, routerName,
 384                     GroupTypes.GroupAll, null);
 385             LOG.info("NAT Service : Removing NAPT Group in new naptSwitch {}", naptSwitch);
 386             mdsalManager.removeGroup(groupEntity);
 387         } catch (Exception ex) {
 388             LOG.debug("NAT Service : Failed to remove group in new naptSwitch {} : {}",groupEntity,ex);
 389         }
 390         return true;
 391     }
 392
 393     private String getExtNetworkVpnName(long routerId) {
 394         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 395         if(networkId == null) {
 396             LOG.error("networkId is null for the router ID {}", routerId);
 397         } else {
 398             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId, LOG);
 399             if (vpnName != null) {
 400                 LOG.debug("retrieved vpn name {} associated with ext nw {} in router {}",
 401                         vpnName,networkId,routerId);
 402                 return vpnName;
 403             } else {
 404                 LOG.error("No VPN associated with ext nw {} belonging to routerId {}",
 405                         networkId, routerId);
 406             }
 407         }
 408         return null;
 409     }
 410
 411     public void updateNaptSwitchBucketStatus(String routerName, BigInteger naptSwitch) {
 412         LOG.debug("updateNaptSwitchBucketStatus method is called");
 413
 414         List<BigInteger> dpnList = naptSwitchSelector.getDpnsForVpn(routerName);
 415         //List<BigInteger> dpnList = getDpnListForRouter(routerName);
 416         if (dpnList == null || dpnList.isEmpty()) {
 417             LOG.debug("No switches found for router {}",routerName);
 418             return;
 419         }
 420         for (BigInteger dpn : dpnList) {
 421             if (!dpn.equals(naptSwitch)) {
 422                 LOG.debug("Updating SNAT_TABLE missentry for DpnId {} which is not naptSwitch for router {}",dpn,routerName);
 423                 List<BucketInfo> bucketInfoList = handleGroupInNeighborSwitches(dpn, routerName, naptSwitch);
 424                 if (bucketInfoList == null) {
 425                     LOG.debug("Failed to populate bucketInfo for orinaryswitch {} whose naptSwitch {} for router {} ",
 426                             dpn,naptSwitch,routerName);
 427                     return;
 428                 }
 429                 modifySnatGroupEntry(dpn, bucketInfoList, routerName);
 430             }
 431         }
 432     }
 433
 434     private boolean handleNatFlowsInNewNaptSwitch(Long routerId,BigInteger oldNaptSwitch, BigInteger newNaptSwitch,Long routerVpnId) {
 435         LOG.debug("Proceeding to install flows in newNaptSwitch {} for routerId {}", newNaptSwitch,routerId);
 436         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker,routerId);
 437         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 438             LOG.debug("No Internal Ip Port mapping associated to router {}, no flows need to be installed in" +
 439                     "newNaptSwitch {}", routerId, newNaptSwitch);
 440             return true;
 441         }
 442         //getvpnId
 443         Long vpnId = getVpnIdForRouter(routerId);
 444         if (vpnId == NatConstants.INVALID_ID) {
 445             LOG.error("Invalid vpnId for routerId {}",routerId);
 446             return false;
 447         }
 448         Long bgpVpnId;
 449         if(routerId.equals(routerVpnId)) {
 450             bgpVpnId = NatConstants.INVALID_ID;
 451         } else {
 452             bgpVpnId = routerVpnId;
 453         }
 454         LOG.debug("retrieved bgpVpnId {} for router {}",bgpVpnId,routerId);
 455         for (IntextIpProtocolType protocolType : ipPortMapping.getIntextIpProtocolType()) {
 456             if (protocolType.getIpPortMap() == null || protocolType.getIpPortMap().isEmpty()) {
 457                 LOG.debug("No {} session associated to router {}", protocolType.getProtocol(), routerId);
 458                 return true;
 459             }
 460             for (IpPortMap intIpPortMap : protocolType.getIpPortMap()) {
 461                 String internalIpAddress = intIpPortMap.getIpPortInternal().split(":")[0];
 462                 String intportnum = intIpPortMap.getIpPortInternal().split(":")[1];
 463
 464                 //Get the external IP address and the port from the model
 465                 NAPTEntryEvent.Protocol proto = protocolType.getProtocol().toString().equals(ProtocolTypes.TCP.toString())
 466                         ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP;
 467                 IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
 468                         internalIpAddress, intportnum, proto);
 469                 if (ipPortExternal == null) {
 470                     LOG.debug("External Ipport mapping is not found for internalIp {} with port {}", internalIpAddress, intportnum);
 471                     continue;
 472                 }
 473                 String externalIpAddress = ipPortExternal.getIpAddress();
 474                 Integer extportNumber = ipPortExternal.getPortNum();
 475                 LOG.debug("ExternalIPport {}:{} mapping for internal ipport {}:{}",externalIpAddress,extportNumber,
 476                         internalIpAddress,intportnum);
 477
 478                 SessionAddress sourceAddress = new SessionAddress(internalIpAddress,Integer.valueOf(intportnum));
 479                 SessionAddress externalAddress = new SessionAddress(externalIpAddress,extportNumber);
 480
 481                 //checking naptSwitch status before installing flows
 482                 if(getSwitchStatus(newNaptSwitch)) {
 483                     //Install the flow in newNaptSwitch Outbound NAPT table.
 484                     try {
 485                         NaptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 486                                 vpnId,  routerId, bgpVpnId, sourceAddress, externalAddress, proto);
 487                     } catch (Exception ex) {
 488                         LOG.error("Failed to add flow in OUTBOUND_NAPT_TABLE for routerid {} dpnId {} ipport {}:{} proto {}" +
 489                                 "extIpport {}:{} BgpVpnId {} - {}", routerId, newNaptSwitch, internalIpAddress
 490                                 , intportnum, proto, externalAddress, extportNumber,bgpVpnId,ex);
 491                         return false;
 492                     }
 493                     LOG.debug("Successfully installed a flow in SecondarySwitch {} Outbound NAPT table for router {} " +
 494                             "ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}", newNaptSwitch,routerId, internalIpAddress
 495                             , intportnum, proto, externalAddress, extportNumber,bgpVpnId);
 496                     //Install the flow in newNaptSwitch Inbound NAPT table.
 497                     try {
 498                         NaptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 499                                 vpnId, routerId, bgpVpnId, externalAddress, sourceAddress, proto);
 500                     } catch (Exception ex) {
 501                         LOG.error("Failed to add flow in INBOUND_NAPT_TABLE for routerid {} dpnId {} extIpport{}:{} proto {} " +
 502                                         "ipport {}:{} BgpVpnId {}", routerId, newNaptSwitch, externalAddress, extportNumber, proto,
 503                                 internalIpAddress, intportnum,bgpVpnId);
 504                         return false;
 505                     }
 506                     LOG.debug("Successfully installed a flow in SecondarySwitch {} Inbound NAPT table for router {} " +
 507                             "ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}", newNaptSwitch,routerId, internalIpAddress
 508                             , intportnum, proto, externalAddress, extportNumber,bgpVpnId);
 509
 510                 } else {
 511                     LOG.error("NewNaptSwitch {} gone down while installing flows from oldNaptswitch {}",
 512                             newNaptSwitch,oldNaptSwitch);
 513                     return false;
 514                 }
 515             }
 516         }
 517         return true;
 518     }
 519
 520     private Long getVpnIdForRouter(Long routerId) {
 521         try {
 522             //getvpnId
 523             Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 524             if (networkId == null) {
 525                 LOG.debug("network is not associated to router {}", routerId);
 526             } else {
 527                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 528                 if (vpnUuid == null) {
 529                     LOG.debug("vpn is not associated for network {} in router {}", networkId, routerId);
 530                 } else {
 531                     Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 532                     if (vpnId > 0) {
 533                         LOG.debug("retrieved vpnId {} for router {}",vpnId,routerId);
 534                         return vpnId;
 535                     } else {
 536                         LOG.debug("retrieved invalid vpn Id");
 537                     }
 538                 }
 539             }
 540         } catch (Exception ex){
 541             LOG.debug("Exception while retrieving vpnId for router {} - {}", routerId, ex);
 542         }
 543         return NatConstants.INVALID_ID;
 544     }
 545
 546     public boolean getSwitchStatus(BigInteger switchId){
 547         NodeId nodeId = new NodeId("openflow:" + switchId);
 548         LOG.debug("Querying switch with dpnId {} is up/down", nodeId);
 549         InstanceIdentifier<Node> nodeInstanceId = InstanceIdentifier.builder(Nodes.class)
 550                 .child(Node.class, new NodeKey(nodeId)).build();
 551         Optional<Node> nodeOptional = NatUtil.read(dataBroker,LogicalDatastoreType.OPERATIONAL,nodeInstanceId);
 552         if (nodeOptional.isPresent()) {
 553             LOG.debug("Switch {} is up", nodeId);
 554             return true;
 555         }
 556         LOG.debug("Switch {} is down", nodeId);
 557         return false;
 558     }
 559
 560     public List<BucketInfo> handleGroupInPrimarySwitch() {
 561         List<BucketInfo> listBucketInfo = new ArrayList<>();
 562         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 563         listActionInfoPrimary.add(new ActionInfo(ActionType.nx_resubmit,
 564                 new String[]{String.valueOf(NwConstants.INTERNAL_TUNNEL_TABLE)}));
 565         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 566         listBucketInfo.add(bucketPrimary);
 567         return listBucketInfo;
 568     }
 569
 570     public List<BucketInfo> handleGroupInNeighborSwitches(BigInteger dpnId, String routerName, BigInteger naptSwitch) {
 571         List<BucketInfo> listBucketInfo = new ArrayList<>();
 572         String ifNamePrimary;
 573         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
 574         if (routerId == NatConstants.INVALID_ID) {
 575             LOG.error("Invalid routerId returned for routerName {}",routerName);
 576             return listBucketInfo;
 577         }
 578         ifNamePrimary = getTunnelInterfaceName(dpnId, naptSwitch);
 579         if (ifNamePrimary != null) {
 580             LOG.debug("TunnelInterface {} between ordinary switch {} and naptSwitch {}",ifNamePrimary,dpnId,naptSwitch);
 581             List<ActionInfo> listActionInfoPrimary = NatUtil.getEgressActionsForInterface(interfaceManager, ifNamePrimary, routerId);
 582             BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 583             listBucketInfo.add(bucketPrimary);
 584         } else {
 585             LOG.debug("No TunnelInterface between ordinary switch {} and naptSwitch {}",dpnId,naptSwitch);
 586         }
 587         return listBucketInfo;
 588     }
 589
 590     protected void installSnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 591         GroupEntity groupEntity = null;
 592         try {
 593             long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 594             LOG.debug("install SnatMissEntry for groupId {} for dpnId {} for router {}", groupId, dpnId,routerName);
 595             groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, routerName,
 596                     GroupTypes.GroupAll, bucketInfo);
 597             mdsalManager.installGroup(groupEntity);
 598             LOG.debug("installed the SNAT to NAPT GroupEntity:{}", groupEntity);
 599         } catch (Exception ex) {
 600             LOG.error("Failed to install group for groupEntity {} : {}",groupEntity,ex);
 601         }
 602     }
 603
 604     private void modifySnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 605         installSnatGroupEntry(dpnId,bucketInfo,routerName);
 606         LOG.debug("modified SnatMissEntry for dpnId {} of router {}",dpnId,routerName);
 607     }
 608
 609     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 610         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 611         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 612
 613         try {
 614             Future<RpcResult<GetTunnelInterfaceNameOutput>> result = itmManager.getTunnelInterfaceName(
 615                     new GetTunnelInterfaceNameInputBuilder().setSourceDpid(srcDpId).setDestinationDpid(dstDpId)
 616                             .setTunnelType(tunType).build());
 617             rpcResult = result.get();
 618             if(!rpcResult.isSuccessful()) {
 619                 tunType = TunnelTypeGre.class;
 620                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 621                         .setSourceDpid(srcDpId)
 622                         .setDestinationDpid(dstDpId)
 623                         .setTunnelType(tunType)
 624                         .build());
 625                 rpcResult = result.get();
 626                 if(!rpcResult.isSuccessful()) {
 627                     LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 628                 } else {
 629                     return rpcResult.getResult().getInterfaceName();
 630                 }
 631                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 632             } else {
 633                 return rpcResult.getResult().getInterfaceName();
 634             }
 635         } catch (InterruptedException | ExecutionException e) {
 636             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {} :",
 637                     srcDpId, dstDpId, e);
 638         }
 639
 640         return null;
 641     }
 642
 643     public boolean updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
 644         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().setKey(new RouterToNaptSwitchKey(routerName))
 645                 .setPrimarySwitchId(naptSwitchId).build();
 646         try {
 647             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 648                     NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
 649         } catch (Exception ex) {
 650             LOG.error("Failed to write naptSwitch {} for router {} in ds",
 651                     naptSwitchId,routerName);
 652             return false;
 653         }
 654         LOG.debug("Successfully updated naptSwitch {} for router {} in ds",
 655                 naptSwitchId,routerName);
 656         return true;
 657     }
 658
 659     public FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long groupId, long routerVpnId, int addordel) {
 660
 661         FlowEntity flowEntity;
 662         List<MatchInfo> matches = new ArrayList<MatchInfo>();
 663         matches.add(new MatchInfo(MatchFieldType.eth_type,
 664                 new long[]{ 0x0800L }));
 665         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 666                 MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID }));
 667
 668         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 669
 670         if (addordel == NatConstants.ADD_FLOW) {
 671             List<InstructionInfo> instructions = new ArrayList<InstructionInfo>();
 672             List<ActionInfo> actionsInfo = new ArrayList<ActionInfo>();
 673
 674             ActionInfo actionSetField = new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] {
 675                     BigInteger.valueOf(routerVpnId)}) ;
 676             actionsInfo.add(actionSetField);
 677             LOG.debug("Setting the tunnel to the list of action infos {}", actionsInfo);
 678             actionsInfo.add(new ActionInfo(ActionType.group, new String[] {String.valueOf(groupId)}));
 679             instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfo));
 680
 681             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 682                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 683                     NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 684         } else {
 685             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 686                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 687                     NwConstants.COOKIE_SNAT_TABLE, matches, null);
 688         }
 689         return flowEntity;
 690     }
 691
 692     public FlowEntity buildSnatFlowEntityForNaptSwitch(BigInteger dpId, String routerName, long routerVpnId, int addordel) {
 693
 694         FlowEntity flowEntity;
 695         List<MatchInfo> matches = new ArrayList<>();
 696         matches.add(new MatchInfo(MatchFieldType.eth_type,
 697                 new long[]{ 0x0800L }));
 698         matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] {
 699                 MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID }));
 700
 701         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 702
 703         if (addordel == NatConstants.ADD_FLOW) {
 704             List<InstructionInfo> instructions = new ArrayList<>();
 705
 706             instructions.add(new InstructionInfo(InstructionType.goto_table, new long[]
 707                     { NwConstants.OUTBOUND_NAPT_TABLE }));
 708
 709             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 710                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 711                     NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 712         } else {
 713             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 714                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 715                     NwConstants.COOKIE_SNAT_TABLE, matches, null);
 716         }
 717         return flowEntity;
 718     }
 719
 720     private String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 721         return new StringBuilder().append(NatConstants.SNAT_FLOWID_PREFIX).append(dpnId).append(NatConstants.FLOWID_SEPARATOR).
 722                 append(tableId).append(NatConstants.FLOWID_SEPARATOR).append(routerID).toString();
 723     }
 724
 725     protected void installSnatFlows(String routerName,Long routerId,BigInteger naptSwitch,Long routerVpnId) {
 726
 727         if(routerId.equals(routerVpnId)) {
 728             LOG.debug("Installing flows for router with internalvpnId");
 729             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 730             LOG.debug("installTerminatingServiceTblEntry in naptswitch with dpnId {} for routerName {} with routerId {}",
 731                     naptSwitch, routerName,routerId);
 732             externalRouterListener.installTerminatingServiceTblEntry(naptSwitch, routerName);
 733
 734             //Install default flows punting to controller in table 46(OutBoundNapt table)
 735             LOG.debug("installOutboundMissEntry in naptswitch with dpnId {} for routerName {} with routerId {}",
 736                     naptSwitch, routerName, routerId);
 737             externalRouterListener.createOutboundTblEntry(naptSwitch, routerId);
 738
 739             //Table 47 point to table 21 for inbound traffic
 740             LOG.debug("installNaptPfibEntry in naptswitch with dpnId {} for router {}", naptSwitch, routerId);
 741             externalRouterListener.installNaptPfibEntry(naptSwitch, routerId);
 742         } else {
 743             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 744             LOG.debug("installTerminatingServiceTblEntry in naptswitch with dpnId {} for routerName {} with BgpVpnId {}",
 745                     naptSwitch, routerName, routerVpnId);
 746             externalRouterListener.installTerminatingServiceTblEntryWithUpdatedVpnId(naptSwitch, routerName, routerVpnId);
 747
 748             //Install default flows punting to controller in table 46(OutBoundNapt table)
 749             LOG.debug("installOutboundMissEntry in naptswitch with dpnId {} for routerName {} with BgpVpnId {}",
 750                     naptSwitch, routerName, routerVpnId);
 751             externalRouterListener.createOutboundTblEntryWithBgpVpn(naptSwitch, routerId, routerVpnId);
 752
 753             //Table 47 point to table 21 for inbound traffic
 754             LOG.debug("installNaptPfibEntry in naptswitch with dpnId {} for router {} with BgpVpnId {}",
 755                     naptSwitch, routerId, routerVpnId);
 756             externalRouterListener.installNaptPfibEntryWithBgpVpn(naptSwitch, routerId, routerVpnId);
 757         }
 758
 759         String vpnName = getExtNetworkVpnName(routerId);
 760         if(vpnName != null) {
 761             //Table 47 point to table 21 for outbound traffic
 762             long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 763             if(vpnId > 0) {
 764                 LOG.debug("installNaptPfibEntry fin naptswitch with dpnId {} for BgpVpnId {}", naptSwitch, vpnId);
 765                 externalRouterListener.installNaptPfibEntry(naptSwitch, vpnId);
 766             } else {
 767                 LOG.debug("Associated BgpvpnId not found for router {}",routerId);
 768             }
 769
 770             //Install Fib entries for ExternalIps & program 36 -> 44
 771             List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 772             if (externalIps != null) {
 773                 for (String externalIp : externalIps) {
 774                     LOG.debug("advToBgpAndInstallFibAndTsFlows in naptswitch id {} with vpnName {} and externalIp {}",
 775                             naptSwitch, vpnName, externalIp);
 776                     externalRouterListener.advToBgpAndInstallFibAndTsFlows(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 777                             vpnName, routerId, externalIp, vpnService, fibService, bgpManager, dataBroker, LOG);
 778                     LOG.debug("Successfully added fib entries in naptswitch {} for router {} with external IP {}", naptSwitch,
 779                             routerId, externalIp);
 780                 }
 781             } else {
 782                 LOG.debug("External Ip not found for routerId {}",routerId);
 783             }
 784         } else {
 785             LOG.debug("Associated vpnName not found for router {}",routerId);
 786         }
 787     }
 788
 789     protected void bestEffortDeletion(long routerId,String routerName,HashMap<String,Long> externalIpLabel) {
 790         List<String> newExternalIps = NatUtil.getExternalIpsForRouter(dataBroker,routerId);
 791         if (newExternalIps != null && externalIpsCache != null) {
 792             Set<String> originalSubnetIds = Sets.newHashSet(externalIpsCache);
 793             Set<String> updatedSubnetIds = Sets.newHashSet(newExternalIps);
 794             Sets.SetView<String> removeExternalIp = Sets.difference(originalSubnetIds, updatedSubnetIds);
 795             if (removeExternalIp.isEmpty()) {
 796                 LOG.debug("No external Ip needed to be removed in bestEffortDeletion method for router {}",routerName);
 797                 return;
 798             }
 799             String vpnName = getExtNetworkVpnName(routerId);
 800             if (vpnName == null) {
 801                 LOG.debug("Vpn is not associated to externalN/w of router {}",routerName);
 802                 return;
 803             }
 804             if (externalIpLabel == null || externalIpLabel.size() == 0) {
 805                 LOG.debug("ExternalIpLabel map is empty for router {}",routerName);
 806                 return;
 807             }
 808             BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
 809             if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 810                 LOG.debug("No naptSwitch is selected for router {}", routerName);
 811                 return;
 812             }
 813             Long label;
 814             for (String externalIp : removeExternalIp) {
 815                 if (externalIpLabel.containsKey(externalIp)) {
 816                     label = externalIpLabel.get(externalIp);
 817                     LOG.debug("Label {} for ExternalIp {} for router {}",label,externalIp,routerName);
 818                 } else {
 819                     LOG.debug("Label for ExternalIp {} is not found for router {}",externalIp,routerName);
 820                     continue;
 821                 }
 822                 externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 823                 externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerId, externalIp, vpnName,label);
 824                 LOG.debug("Successfully removed fib entries in switch {} for router {} and externalIps {}",
 825                             naptSwitch, routerId, externalIp);
 826             }
 827         } else {
 828             LOG.debug("No external IP found for router {}",routerId);
 829         }
 830     }
 831 }