natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/NaptSwitchHA.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import java.math.BigInteger;
  12 import java.util.ArrayList;
  13 import java.util.Collection;
  14 import java.util.HashSet;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Map.Entry;
  18 import java.util.Set;
  19 import java.util.concurrent.ExecutionException;
  20 import java.util.concurrent.Future;
  21 import javax.inject.Inject;
  22 import javax.inject.Singleton;
  23 import org.eclipse.jdt.annotation.NonNull;
  24 import org.eclipse.jdt.annotation.Nullable;
  25 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  26 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  28 import org.opendaylight.genius.infra.Datastore.Configuration;
  29 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  30 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  31 import org.opendaylight.genius.mdsalutil.ActionInfo;
  32 import org.opendaylight.genius.mdsalutil.BucketInfo;
  33 import org.opendaylight.genius.mdsalutil.FlowEntity;
  34 import org.opendaylight.genius.mdsalutil.GroupEntity;
  35 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  36 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  37 import org.opendaylight.genius.mdsalutil.MatchInfo;
  38 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  48 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  49 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  50 import org.opendaylight.netvirt.natservice.api.SnatServiceManager;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  81 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  82 import org.opendaylight.yangtools.yang.common.RpcResult;
  83 import org.slf4j.Logger;
  84 import org.slf4j.LoggerFactory;
  85
  86 @Singleton
  87 public class NaptSwitchHA {
  88     private static final Logger LOG = LoggerFactory.getLogger(NaptSwitchHA.class);
  89     private final DataBroker dataBroker;
  90     private final IMdsalApiManager mdsalManager;
  91     private final ItmRpcService itmManager;
  92     private final OdlInterfaceRpcService odlInterfaceRpcService;
  93     private final IdManagerService idManager;
  94     private final NAPTSwitchSelector naptSwitchSelector;
  95     private final ExternalRoutersListener externalRouterListener;
  96     private final NaptEventHandler naptEventHandler;
  97     private final IFibManager fibManager;
  98     private final IElanService elanManager;
  99     private final EvpnNaptSwitchHA evpnNaptSwitchHA;
 100     private final SnatServiceManager natServiceManager;
 101     private final NatMode natMode;
 102     private final IInterfaceManager interfaceManager;
 103     private final NatOverVxlanUtil natOverVxlanUtil;
 104
 105     private volatile Collection<String> externalIpsCache;
 106
 107     @Inject
 108     public NaptSwitchHA(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 109                         final ExternalRoutersListener externalRouterListener,
 110                         final ItmRpcService itmManager,
 111                         final OdlInterfaceRpcService odlInterfaceRpcService,
 112                         final IdManagerService idManager,
 113                         final NAPTSwitchSelector naptSwitchSelector,
 114                         final IFibManager fibManager,
 115                         final EvpnNaptSwitchHA evpnNaptSwitchHA,
 116                         final IElanService elanManager,
 117                         final SnatServiceManager natServiceManager,
 118                         final NatserviceConfig config,
 119                         final NaptEventHandler naptEventHandler,
 120                         final IInterfaceManager interfaceManager,
 121                         final NatOverVxlanUtil natOverVxlanUtil) {
 122         this.dataBroker = dataBroker;
 123         this.mdsalManager = mdsalManager;
 124         this.externalRouterListener = externalRouterListener;
 125         this.itmManager = itmManager;
 126         this.odlInterfaceRpcService = odlInterfaceRpcService;
 127         this.idManager = idManager;
 128         this.naptSwitchSelector = naptSwitchSelector;
 129         this.naptEventHandler = naptEventHandler;
 130         this.fibManager = fibManager;
 131         this.evpnNaptSwitchHA = evpnNaptSwitchHA;
 132         this.elanManager = elanManager;
 133         this.natServiceManager = natServiceManager;
 134         this.interfaceManager = interfaceManager;
 135         if (config != null) {
 136             this.natMode = config.getNatMode();
 137         } else {
 138             this.natMode = NatMode.Controller;
 139         }
 140         this.natOverVxlanUtil = natOverVxlanUtil;
 141     }
 142
 143     protected void removeSnatFlowsInOldNaptSwitch(String routerName, Long routerId, BigInteger naptSwitch,
 144                                                   @Nullable Map<String, Long> externalIpmap,
 145                                                   TypedReadWriteTransaction<Configuration> confTx)
 146             throws ExecutionException, InterruptedException {
 147
 148         //remove SNAT flows in old NAPT SWITCH
 149         Uuid networkId = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 150         String vpnName = getExtNetworkVpnName(routerName, networkId);
 151         if (vpnName == null) {
 152             LOG.error("removeSnatFlowsInOldNaptSwitch : Vpn is not associated to externalN/w of router {}",
 153                 routerName);
 154             return;
 155         }
 156         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, networkId);
 157         if (extNwProvType == null) {
 158             LOG.error("removeSnatFlowsInOldNaptSwitch : Unable to retrieve the External Network Provider Type "
 159                 + "for Router {}", routerName);
 160             return;
 161         }
 162         if (extNwProvType == ProviderTypes.VXLAN) {
 163             evpnNaptSwitchHA.evpnRemoveSnatFlowsInOldNaptSwitch(routerName, routerId, vpnName, naptSwitch, confTx);
 164         } else {
 165             //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table
 166             long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 167                     idManager, routerId, routerName);
 168             String tsFlowRef = externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE,
 169                 tunnelId);
 170             FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.INTERNAL_TUNNEL_TABLE,
 171                 tsFlowRef);
 172
 173             LOG.info("removeSnatFlowsInOldNaptSwitch : Remove the flow in table {} for the old napt switch "
 174                 + "with the DPN ID {} and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, naptSwitch, routerId);
 175             mdsalManager.removeFlow(confTx, tsNatFlowEntity);
 176         }
 177         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
 178             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
 179             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, naptSwitch);
 180         }
 181         //Remove the Outbound flow entry which forwards the packet to Outbound NAPT Table
 182         LOG.info("Remove the flow in table {} for the old napt switch with the DPN ID {} and router ID {}",
 183             NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId);
 184
 185         String outboundTcpNatFlowRef = externalRouterListener.getFlowRefOutbound(naptSwitch,
 186             NwConstants.OUTBOUND_NAPT_TABLE, routerId, NwConstants.IP_PROT_TCP);
 187         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch,
 188             NwConstants.OUTBOUND_NAPT_TABLE, outboundTcpNatFlowRef);
 189         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
 190
 191         String outboundUdpNatFlowRef = externalRouterListener.getFlowRefOutbound(naptSwitch,
 192             NwConstants.OUTBOUND_NAPT_TABLE, routerId, NwConstants.IP_PROT_UDP);
 193         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(naptSwitch,
 194             NwConstants.OUTBOUND_NAPT_TABLE, outboundUdpNatFlowRef);
 195         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
 196
 197         String icmpDropFlowRef = externalRouterListener.getFlowRefOutbound(naptSwitch,
 198             NwConstants.OUTBOUND_NAPT_TABLE, routerId, NwConstants.IP_PROT_ICMP);
 199         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 200             icmpDropFlowRef);
 201         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
 202
 203         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
 204         // External Subnet Vpn Id.
 205         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
 206             routerName);
 207         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
 208             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
 209             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
 210                 dataBroker, externalSubnetId, routerName, naptSwitch)) {
 211                 String natPfibSubnetFlowRef = externalRouterListener.getFlowRefTs(naptSwitch,
 212                     NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
 213                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE,
 214                     natPfibSubnetFlowRef);
 215                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
 216                 LOG.debug("removeSnatFlowsInOldNaptSwitch : Removed the flow in table {} with external subnet "
 217                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
 218                     subnetVpnId, naptSwitch);
 219             }
 220         }
 221
 222         // Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for inbound traffic
 223         // matching on the router ID.
 224         String naptPFibflowRef =
 225             externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, routerId);
 226         FlowEntity naptPFibFlowEntity =
 227             NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE, naptPFibflowRef);
 228         LOG.info("removeSnatFlowsInOldNaptSwitch : Remove the flow in table {} for the old napt switch "
 229             + "with the DPN ID {} and router ID {}", NwConstants.NAPT_PFIB_TABLE, naptSwitch, routerId);
 230         mdsalManager.removeFlow(confTx, naptPFibFlowEntity);
 231
 232         // Remove the NAPT_PFIB_TABLE(47) flow entry forwards the packet to Fib Table for outbound traffic
 233         // matching on the vpn ID.
 234         boolean switchSharedByRouters = false;
 235         Uuid extNetworkId = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 236         if (extNetworkId != null && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(
 237             dataBroker, networkId, routerName, naptSwitch)) {
 238             List<String> routerNamesAssociated = getRouterIdsForExtNetwork(extNetworkId);
 239             for (String routerNameAssociated : routerNamesAssociated) {
 240                 if (!routerNameAssociated.equals(routerName)) {
 241                     Long routerIdAssociated = NatUtil.getVpnId(dataBroker, routerNameAssociated);
 242                     BigInteger naptDpn = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerNameAssociated);
 243                     if (naptDpn != null && naptDpn.equals(naptSwitch)) {
 244                         LOG.debug("removeSnatFlowsInOldNaptSwitch : Napt switch {} is also acting as primary "
 245                             + "for router {}", naptSwitch, routerIdAssociated);
 246                         switchSharedByRouters = true;
 247                         break;
 248                     }
 249                 }
 250             }
 251             if (!switchSharedByRouters) {
 252                 Long vpnId = getVpnIdForRouter(routerId, extNetworkId);
 253                 if (vpnId != NatConstants.INVALID_ID) {
 254                     String naptFibflowRef =
 255                         externalRouterListener.getFlowRefTs(naptSwitch, NwConstants.NAPT_PFIB_TABLE, vpnId);
 256                     FlowEntity naptFibFlowEntity =
 257                         NatUtil.buildFlowEntity(naptSwitch, NwConstants.NAPT_PFIB_TABLE, naptFibflowRef);
 258                     LOG.info("removeSnatFlowsInOldNaptSwitch : Remove the flow in table {} for the old napt switch"
 259                         + " with the DPN ID {} and vpnId {}", NwConstants.NAPT_PFIB_TABLE, naptSwitch, vpnId);
 260                     mdsalManager.removeFlow(confTx, naptFibFlowEntity);
 261                 } else {
 262                     LOG.error("removeSnatFlowsInOldNaptSwitch : Invalid vpnId retrieved for routerId {}",
 263                         routerId);
 264                     return;
 265                 }
 266             }
 267         }
 268
 269         //Remove Fib entries,tables 20->44 ,36-> 44
 270         String gwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
 271         if (externalIpmap != null && !externalIpmap.isEmpty()) {
 272             for (Entry<String, Long> entry : externalIpmap.entrySet()) {
 273                 String externalIp = entry.getKey();
 274                 Long label = entry.getValue();
 275                 externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerName, routerId, externalIp, vpnName,
 276                     extNetworkId, label, gwMacAddress, true, confTx);
 277                 LOG.debug("removeSnatFlowsInOldNaptSwitch : Successfully removed fib entries in old naptswitch {} "
 278                     + "for router {} and externalIps {} label {}", naptSwitch, routerId, externalIp, label);
 279             }
 280         } else {
 281             List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 282             if (networkId != null) {
 283                 externalRouterListener.clearFibTsAndReverseTraffic(naptSwitch, routerId, networkId,
 284                     externalIps, null, gwMacAddress, confTx);
 285                 LOG.debug(
 286                     "removeSnatFlowsInOldNaptSwitch : Successfully removed fib entries in old naptswitch {} for "
 287                         + "router {} with networkId {} and externalIps {}", naptSwitch, routerId, networkId,
 288                     externalIps);
 289             } else {
 290                 LOG.debug("removeSnatFlowsInOldNaptSwitch : External network not associated to router {}",
 291                     routerId);
 292             }
 293             externalRouterListener.removeNaptFibExternalOutputFlows(routerId, naptSwitch, extNetworkId,
 294                 externalIps, confTx);
 295         }
 296
 297         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
 298         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
 299         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null
 300             || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 301             LOG.warn("removeSnatFlowsInOldNaptSwitch : No Internal Ip Port mapping associated to router {}, "
 302                 + "no flows need to be removed in oldNaptSwitch {}", routerId, naptSwitch);
 303             return;
 304         }
 305         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
 306         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
 307         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
 308             if (intextIpProtocolType.getIpPortMap() == null || intextIpProtocolType.getIpPortMap().isEmpty()) {
 309                 LOG.debug("removeSnatFlowsInOldNaptSwitch : No {} session associated to router {},"
 310                         + "no flows need to be removed in oldNaptSwitch {}",
 311                     intextIpProtocolType.getProtocol(), routerId, naptSwitch);
 312                 break;
 313             }
 314             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
 315             for (IpPortMap ipPortMap : ipPortMaps) {
 316                 String ipPortInternal = ipPortMap.getIpPortInternal();
 317                 String[] ipPortParts = ipPortInternal.split(":");
 318                 if (ipPortParts.length != 2) {
 319                     LOG.error("removeSnatFlowsInOldNaptSwitch : Unable to retrieve the Internal IP and port");
 320                     continue;
 321                 }
 322                 String internalIp = ipPortParts[0];
 323                 String internalPort = ipPortParts[1];
 324
 325                 //Build and remove flow in outbound NAPT table
 326                 String switchFlowRef =
 327                     NatUtil.getNaptFlowRef(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE, String.valueOf(routerId),
 328                         internalIp, Integer.parseInt(internalPort));
 329                 FlowEntity outboundNaptFlowEntity =
 330                     NatUtil.buildFlowEntity(naptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 331                         cookieSnatFlow, switchFlowRef);
 332
 333                 LOG.info("removeSnatFlowsInOldNaptSwitch : Remove the flow in table {} for old napt switch "
 334                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, naptSwitch, routerId);
 335                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
 336
 337                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
 338                 if (ipPortExternal == null) {
 339                     LOG.debug(
 340                         "removeSnatFlowsInOldNaptSwitch : External Ipport mapping not found for internalIp {} "
 341                             + "with port {} for router {}", internalIp, internalPort, routerId);
 342                     continue;
 343                 }
 344                 String externalIp = ipPortExternal.getIpAddress();
 345                 int externalPort = ipPortExternal.getPortNum();
 346
 347                 //Build and remove flow in  inbound NAPT table
 348                 switchFlowRef =
 349                     NatUtil.getNaptFlowRef(naptSwitch, NwConstants.INBOUND_NAPT_TABLE, String.valueOf(routerId),
 350                         externalIp, externalPort);
 351                 FlowEntity inboundNaptFlowEntity =
 352                     NatUtil.buildFlowEntity(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 353                         cookieSnatFlow, switchFlowRef);
 354
 355                 LOG.info(
 356                     "removeSnatFlowsInOldNaptSwitch : Remove the flow in table {} for old napt switch with the "
 357                         + "DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, naptSwitch, routerId);
 358                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
 359             }
 360         }
 361     }
 362
 363     @NonNull
 364     private List<String> getRouterIdsForExtNetwork(Uuid extNetworkId) {
 365         List<String> routerUuidsAsString = new ArrayList<>();
 366         InstanceIdentifier<Networks> extNetwork = InstanceIdentifier.builder(ExternalNetworks.class)
 367             .child(Networks.class, new NetworksKey(extNetworkId)).build();
 368         Optional<Networks> extNetworkData =
 369                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 370                         LogicalDatastoreType.CONFIGURATION, extNetwork);
 371         if (extNetworkData.isPresent()) {
 372             List<Uuid> routerUuids = extNetworkData.get().getRouterIds();
 373             if (routerUuids != null) {
 374                 for (Uuid routerUuid : routerUuids) {
 375                     routerUuidsAsString.add(routerUuid.getValue());
 376                 }
 377             }
 378         }
 379         return routerUuidsAsString;
 380     }
 381
 382     public boolean isNaptSwitchDown(String routerName, Long routerId, BigInteger dpnId, BigInteger naptSwitch,
 383                                     Long routerVpnId, Collection<String> externalIpCache,
 384                                     TypedReadWriteTransaction<Configuration> confTx)
 385             throws ExecutionException, InterruptedException {
 386         return isNaptSwitchDown(routerName, routerId, dpnId, naptSwitch, routerVpnId, externalIpCache, true,
 387                 confTx);
 388     }
 389
 390     // TODO Clean up the exception handling
 391     @SuppressWarnings("checkstyle:IllegalCatch")
 392     public boolean isNaptSwitchDown(String routerName, Long routerId, BigInteger dpnId, BigInteger naptSwitch,
 393                                     Long routerVpnId, Collection<String> externalIpCache, boolean isClearBgpRts,
 394                                     TypedReadWriteTransaction<Configuration> confTx)
 395             throws ExecutionException, InterruptedException {
 396         externalIpsCache = externalIpCache;
 397         if (!naptSwitch.equals(dpnId)) {
 398             LOG.debug("isNaptSwitchDown : DpnId {} is not a naptSwitch {} for Router {}",
 399                     dpnId, naptSwitch, routerName);
 400             return false;
 401         }
 402         LOG.debug("NaptSwitch {} is down for Router {}", naptSwitch, routerName);
 403         if (routerId == NatConstants.INVALID_ID) {
 404             LOG.error("isNaptSwitchDown : Invalid routerId returned for routerName {}", routerName);
 405             return true;
 406         }
 407         Uuid networkId = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 408         String vpnName = getExtNetworkVpnName(routerName, networkId);
 409         //elect a new NaptSwitch
 410         naptSwitch = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 411         if (natMode == NatMode.Conntrack) {
 412             Routers extRouters = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
 413             natServiceManager.notify(confTx, extRouters, null, dpnId, dpnId,
 414                     SnatServiceManager.Action.CNT_ROUTER_ALL_SWITCH_DISBL);
 415             if (extRouters.isEnableSnat()) {
 416                 natServiceManager.notify(confTx, extRouters, null, dpnId, dpnId,
 417                         SnatServiceManager.Action.SNAT_ALL_SWITCH_DISBL);
 418             }
 419             natServiceManager.notify(confTx, extRouters, null, naptSwitch, naptSwitch,
 420                     SnatServiceManager.Action.CNT_ROUTER_ALL_SWITCH_ENBL);
 421             if (extRouters.isEnableSnat()) {
 422                 natServiceManager.notify(confTx, extRouters, null, naptSwitch, naptSwitch,
 423                         SnatServiceManager.Action.SNAT_ALL_SWITCH_ENBL);
 424             }
 425         } else {
 426             if (naptSwitch.equals(BigInteger.ZERO)) {
 427                 LOG.warn("isNaptSwitchDown : No napt switch is elected since all the switches for router {}"
 428                         + " are down. SNAT IS NOT SUPPORTED FOR ROUTER {}", routerName, routerName);
 429                 boolean naptUpdatedStatus = updateNaptSwitch(routerName, naptSwitch);
 430                 if (!naptUpdatedStatus) {
 431                     LOG.debug("isNaptSwitchDown : Failed to update naptSwitch {} for router {} in ds",
 432                             naptSwitch, routerName);
 433                 }
 434                 //clearBgpRoutes
 435                 if (externalIpsCache != null) {
 436                     if (vpnName != null) {
 437                         //List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 438                         //if (externalIps != null) {
 439                         if (isClearBgpRts) {
 440                             LOG.debug("isNaptSwitchDown : Clearing both FIB entries and the BGP routes");
 441                             for (String externalIp : externalIpsCache) {
 442                                 externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 443                             }
 444                         } else {
 445                             LOG.debug("isNaptSwitchDown : Clearing the FIB entries but not the BGP routes");
 446                             String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 447                             for (String externalIp : externalIpsCache) {
 448                                 LOG.debug("isNaptSwitchDown : Removing Fib entry rd {} prefix {}", rd, externalIp);
 449                                 fibManager.removeFibEntry(rd, externalIp, null);
 450                             }
 451                         }
 452                     } else {
 453                         LOG.debug("isNaptSwitchDown : vpn is not associated to extn/w for router {}", routerName);
 454                     }
 455                 } else {
 456                     LOG.debug("isNaptSwitchDown : No ExternalIps found for subnets under router {}, "
 457                             + "no bgp routes need to be cleared", routerName);
 458                 }
 459                 return true;
 460             }
 461             //checking elected switch health status
 462             if (!NatUtil.getSwitchStatus(dataBroker, naptSwitch)) {
 463                 LOG.error("isNaptSwitchDown : Newly elected Napt switch {} for router {} is down",
 464                         naptSwitch, routerName);
 465                 return true;
 466             }
 467             LOG.debug("isNaptSwitchDown : New NaptSwitch {} is up for Router {} and can proceed for flow installation",
 468                     naptSwitch, routerName);
 469             //update napt model for new napt switch
 470             boolean naptUpdated = updateNaptSwitch(routerName, naptSwitch);
 471             if (naptUpdated) {
 472                 //update group of ordinary switch point to naptSwitch tunnel port
 473                 updateNaptSwitchBucketStatus(routerName, routerId, naptSwitch);
 474             } else {
 475                 LOG.error("isNaptSwitchDown : Failed to update naptSwitch model for newNaptSwitch {} for router {}",
 476                         naptSwitch, routerName);
 477             }
 478
 479             //update table26 forward packets to table46(outbound napt table)
 480             FlowEntity flowEntity =
 481                     buildSnatFlowEntityForNaptSwitch(naptSwitch, routerName, routerVpnId, NatConstants.ADD_FLOW);
 482             if (flowEntity == null) {
 483                 LOG.error("isNaptSwitchDown : Failed to populate flowentity for router {} in naptSwitch {}",
 484                         routerName, naptSwitch);
 485             } else {
 486                 LOG.debug("isNaptSwitchDown : Successfully installed flow in naptSwitch {} for router {}",
 487                         naptSwitch, routerName);
 488                 mdsalManager.addFlow(confTx, flowEntity);
 489             }
 490
 491             installSnatFlows(routerName, routerId, naptSwitch, routerVpnId, confTx);
 492
 493             boolean flowInstalledStatus = handleNatFlowsInNewNaptSwitch(routerName, routerId, dpnId, naptSwitch,
 494                     routerVpnId, networkId);
 495             if (flowInstalledStatus) {
 496                 LOG.debug("isNaptSwitchDown :Installed all active session flows in newNaptSwitch {} for routerName {}",
 497                         naptSwitch, routerName);
 498             } else {
 499                 LOG.error("isNaptSwitchDown : Failed to install flows in newNaptSwitch {} for routerId {}",
 500                         naptSwitch, routerId);
 501             }
 502
 503             //remove group in new naptswitch, coz this switch acted previously as ordinary switch
 504             long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 505             try {
 506                 LOG.info("isNaptSwitchDown : Removing NAPT Group in new naptSwitch {}", naptSwitch);
 507                 mdsalManager.removeGroup(confTx, naptSwitch, groupId);
 508             } catch (Exception ex) {
 509                 LOG.error("isNaptSwitchDown : Failed to remove group in new naptSwitch {}", naptSwitch, ex);
 510             }
 511         }
 512         return true;
 513     }
 514
 515     @Nullable
 516     private String getExtNetworkVpnName(String routerName, Uuid networkId) {
 517         if (networkId == null) {
 518             LOG.error("getExtNetworkVpnName : networkId is null for the router ID {}", routerName);
 519         } else {
 520             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
 521             if (vpnName != null) {
 522                 LOG.debug("getExtNetworkVpnName : retrieved vpn name {} associated with ext nw {} in router {}",
 523                     vpnName, networkId, routerName);
 524                 return vpnName;
 525             } else {
 526                 LOG.error("getExtNetworkVpnName : No VPN associated with ext nw {} belonging to routerId {}",
 527                     networkId, routerName);
 528             }
 529         }
 530         LOG.error("getExtNetworkVpnName : External Network VPN not found for router : {}", routerName);
 531         return null;
 532     }
 533
 534     public void updateNaptSwitchBucketStatus(String routerName, long routerId, BigInteger naptSwitch) {
 535         LOG.debug("updateNaptSwitchBucketStatus : called");
 536
 537         List<BigInteger> dpnList = naptSwitchSelector.getDpnsForVpn(routerName);
 538         //List<BigInteger> dpnList = getDpnListForRouter(routerName);
 539         if (dpnList.isEmpty()) {
 540             LOG.warn("updateNaptSwitchBucketStatus : No switches found for router {}", routerName);
 541             return;
 542         }
 543         for (BigInteger dpn : dpnList) {
 544             if (!dpn.equals(naptSwitch)) {
 545                 LOG.debug("updateNaptSwitchBucketStatus : Updating SNAT_TABLE missentry for DpnId {} "
 546                         + "which is not naptSwitch for router {}", dpn, routerName);
 547                 List<BucketInfo> bucketInfoList = handleGroupInNeighborSwitches(dpn, routerName, routerId, naptSwitch);
 548                 modifySnatGroupEntry(dpn, bucketInfoList, routerName);
 549             }
 550         }
 551     }
 552
 553     // TODO Clean up the exception handling
 554     @SuppressWarnings("checkstyle:IllegalCatch")
 555     private boolean handleNatFlowsInNewNaptSwitch(String routerName, Long routerId, BigInteger oldNaptSwitch,
 556                                                     BigInteger newNaptSwitch, Long routerVpnId, Uuid networkId) {
 557         LOG.debug("handleNatFlowsInNewNaptSwitch : Proceeding to install flows in newNaptSwitch {} for routerId {}",
 558                 newNaptSwitch, routerId);
 559         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
 560         if (ipPortMapping == null || ipPortMapping.getIntextIpProtocolType() == null
 561             || ipPortMapping.getIntextIpProtocolType().isEmpty()) {
 562             LOG.debug("handleNatFlowsInNewNaptSwitch : No Internal Ip Port mapping associated to router {},"
 563                     + "no flows need to be installed in newNaptSwitch {}", routerId, newNaptSwitch);
 564             return true;
 565         }
 566         //getvpnId
 567         Long vpnId = getVpnIdForRouter(routerId, networkId);
 568         if (vpnId == NatConstants.INVALID_ID) {
 569             LOG.error("handleNatFlowsInNewNaptSwitch : Invalid vpnId for routerId {}", routerId);
 570             return false;
 571         }
 572         Long bgpVpnId;
 573         if (routerId.equals(routerVpnId)) {
 574             bgpVpnId = NatConstants.INVALID_ID;
 575         } else {
 576             bgpVpnId = routerVpnId;
 577         }
 578         LOG.debug("handleNatFlowsInNewNaptSwitch : retrieved bgpVpnId {} for router {}", bgpVpnId, routerId);
 579         // Get the External Gateway MAC Address
 580         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
 581         if (extGwMacAddress != null) {
 582             LOG.debug("handleNatFlowsInNewNaptSwitch :External Gateway MAC address {} found for External Router ID {}",
 583                     extGwMacAddress, routerId);
 584         } else {
 585             LOG.error("handleNatFlowsInNewNaptSwitch : No External Gateway MAC address found for External Router ID {}",
 586                     routerId);
 587             return false;
 588         }
 589         for (IntextIpProtocolType protocolType : ipPortMapping.getIntextIpProtocolType()) {
 590             if (protocolType.getIpPortMap() == null || protocolType.getIpPortMap().isEmpty()) {
 591                 LOG.debug("handleNatFlowsInNewNaptSwitch : No {} session associated to router {}",
 592                         protocolType.getProtocol(), routerId);
 593                 return true;
 594             }
 595             for (IpPortMap intIpPortMap : protocolType.getIpPortMap()) {
 596                 String internalIpAddress = intIpPortMap.getIpPortInternal().split(":")[0];
 597                 String intportnum = intIpPortMap.getIpPortInternal().split(":")[1];
 598                 LOG.debug("handleNatFlowsInNewNaptSwitch : Found Internal IP Address {} and Port Number {}",
 599                     internalIpAddress, intportnum);
 600                 //Get the external IP address and the port from the model
 601                 NAPTEntryEvent.Protocol proto =
 602                     protocolType.getProtocol().toString().equals(ProtocolTypes.TCP.toString())
 603                     ? NAPTEntryEvent.Protocol.TCP : NAPTEntryEvent.Protocol.UDP;
 604                 IpPortExternal ipPortExternal = NatUtil.getExternalIpPortMap(dataBroker, routerId,
 605                     internalIpAddress, intportnum, proto);
 606                 if (ipPortExternal == null) {
 607                     LOG.debug("handleNatFlowsInNewNaptSwitch : External Ipport mapping is not found for internalIp {} "
 608                             + "with port {}", internalIpAddress, intportnum);
 609                     continue;
 610                 }
 611                 String externalIpAddress = ipPortExternal.getIpAddress();
 612                 Integer extportNumber = ipPortExternal.getPortNum();
 613                 LOG.debug("handleNatFlowsInNewNaptSwitch : ExternalIPport {}:{} mapping for internal ipport {}:{}",
 614                         externalIpAddress, extportNumber, internalIpAddress, intportnum);
 615
 616                 SessionAddress sourceAddress = new SessionAddress(internalIpAddress, Integer.parseInt(intportnum));
 617                 SessionAddress externalAddress = new SessionAddress(externalIpAddress, extportNumber);
 618
 619                 //checking naptSwitch status before installing flows
 620                 if (NatUtil.getSwitchStatus(dataBroker, newNaptSwitch)) {
 621                     //Install the flow in newNaptSwitch Inbound NAPT table.
 622                     try {
 623                         naptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 624                             vpnId, routerId, bgpVpnId, externalAddress, sourceAddress, proto, extGwMacAddress);
 625                     } catch (RuntimeException ex) {
 626                         LOG.error("handleNatFlowsInNewNaptSwitch : Failed to add flow in INBOUND_NAPT_TABLE for "
 627                                 + "routerid {} dpnId {} extIpport{}:{} proto {} ipport {}:{} BgpVpnId {}",
 628                             routerId, newNaptSwitch, externalAddress, extportNumber, proto,
 629                             internalIpAddress, intportnum, bgpVpnId);
 630                         return false;
 631                     }
 632                     LOG.debug("handleNatFlowsInNewNaptSwitch : Successfully installed a flow in Primary switch {} "
 633                             + "Inbound NAPT table for router {} ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}",
 634                         newNaptSwitch, routerId, internalIpAddress,
 635                         intportnum, proto, externalAddress, extportNumber, bgpVpnId);
 636                     //Install the flow in newNaptSwitch Outbound NAPT table.
 637                     try {
 638                         naptEventHandler.buildAndInstallNatFlows(newNaptSwitch, NwConstants.OUTBOUND_NAPT_TABLE,
 639                             vpnId, routerId, bgpVpnId, sourceAddress, externalAddress, proto, extGwMacAddress);
 640                     } catch (RuntimeException ex) {
 641                         LOG.error("handleNatFlowsInNewNaptSwitch : Failed to add flow in OUTBOUND_NAPT_TABLE for "
 642                                 + "routerid {} dpnId {} ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}",
 643                             routerId, newNaptSwitch, internalIpAddress,
 644                             intportnum, proto, externalAddress, extportNumber, bgpVpnId, ex);
 645                         return false;
 646                     }
 647                     LOG.debug("handleNatFlowsInNewNaptSwitch : Successfully installed a flow in Primary switch {} "
 648                             + "Outbound NAPT table for router {} ipport {}:{} proto {} extIpport {}:{} BgpVpnId {}",
 649                         newNaptSwitch, routerId, internalIpAddress,
 650                         intportnum, proto, externalAddress, extportNumber, bgpVpnId);
 651                 } else {
 652                     LOG.error("handleNatFlowsInNewNaptSwitch : NewNaptSwitch {} gone down while installing flows "
 653                             + "from oldNaptswitch {}", newNaptSwitch, oldNaptSwitch);
 654                     return false;
 655                 }
 656             }
 657         }
 658         return true;
 659     }
 660
 661     // TODO Clean up the exception handling
 662     @SuppressWarnings("checkstyle:IllegalCatch")
 663     private Long getVpnIdForRouter(Long routerId, Uuid networkId) {
 664         try {
 665             //getvpnId
 666             if (networkId == null) {
 667                 LOG.debug("getVpnIdForRouter : network is not associated to router {}", routerId);
 668             } else {
 669                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 670                 if (vpnUuid == null) {
 671                     LOG.debug("getVpnIdForRouter : vpn is not associated for network {} in router {}",
 672                             networkId, routerId);
 673                 } else {
 674                     Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 675                     if (vpnId > 0) {
 676                         LOG.debug("getVpnIdForRouter : retrieved vpnId {} for router {}", vpnId, routerId);
 677                         return vpnId;
 678                     } else {
 679                         LOG.debug("getVpnIdForRouter : retrieved invalid vpn Id");
 680                     }
 681                 }
 682             }
 683         } catch (Exception ex) {
 684             LOG.error("getVpnIdForRouter : Exception while retrieving vpnId for router {}", routerId, ex);
 685         }
 686         return NatConstants.INVALID_ID;
 687     }
 688
 689     public List<BucketInfo> handleGroupInPrimarySwitch() {
 690         List<BucketInfo> listBucketInfo = new ArrayList<>();
 691         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 692         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 693         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 694         listBucketInfo.add(bucketPrimary);
 695         return listBucketInfo;
 696     }
 697
 698     @NonNull
 699     public List<BucketInfo> handleGroupInNeighborSwitches(BigInteger dpnId, String routerName, long routerId,
 700             BigInteger naptSwitch) {
 701         List<BucketInfo> listBucketInfo = new ArrayList<>();
 702         String ifNamePrimary;
 703         if (routerId == NatConstants.INVALID_ID) {
 704             LOG.error("handleGroupInNeighborSwitches : Invalid routerId returned for routerName {}", routerName);
 705             return listBucketInfo;
 706         }
 707         ifNamePrimary = getTunnelInterfaceName(dpnId, naptSwitch);
 708         if (ifNamePrimary != null) {
 709             LOG.debug("handleGroupInNeighborSwitches : TunnelInterface {} between ordinary switch {} and naptSwitch {}",
 710                 ifNamePrimary, dpnId, naptSwitch);
 711             List<ActionInfo> listActionInfoPrimary =
 712                 NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager, interfaceManager,
 713                         ifNamePrimary, routerId, true);
 714             BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 715             listBucketInfo.add(bucketPrimary);
 716         } else {
 717             LOG.debug("handleGroupInNeighborSwitches : No TunnelInterface between ordinary switch {} and naptSwitch {}",
 718                     dpnId, naptSwitch);
 719         }
 720         return listBucketInfo;
 721     }
 722
 723     // TODO Clean up the exception handling
 724     @SuppressWarnings("checkstyle:IllegalCatch")
 725     protected void installSnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 726         GroupEntity groupEntity = null;
 727         try {
 728             long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 729             LOG.debug("installSnatGroupEntry : install SnatMissEntry for groupId {} for dpnId {} for router {}",
 730                     groupId, dpnId, routerName);
 731             groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, routerName,
 732                 GroupTypes.GroupAll, bucketInfo);
 733             mdsalManager.syncInstallGroup(groupEntity);
 734             LOG.debug("installSnatGroupEntry : installed the SNAT to NAPT GroupEntity:{}", groupEntity);
 735         } catch (Exception ex) {
 736             LOG.error("installSnatGroupEntry : Failed to install group for groupEntity {}", groupEntity, ex);
 737         }
 738     }
 739
 740     private void modifySnatGroupEntry(BigInteger dpnId, List<BucketInfo> bucketInfo, String routerName) {
 741         installSnatGroupEntry(dpnId, bucketInfo, routerName);
 742         LOG.debug("modifySnatGroupEntry : modified SnatMissEntry for dpnId {} of router {}", dpnId, routerName);
 743     }
 744
 745     @Nullable
 746     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 747         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 748         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 749
 750         try {
 751             Future<RpcResult<GetTunnelInterfaceNameOutput>> result = itmManager.getTunnelInterfaceName(
 752                 new GetTunnelInterfaceNameInputBuilder().setSourceDpid(srcDpId).setDestinationDpid(dstDpId)
 753                     .setTunnelType(tunType).build());
 754             rpcResult = result.get();
 755             if (!rpcResult.isSuccessful()) {
 756                 tunType = TunnelTypeGre.class;
 757                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 758                     .setSourceDpid(srcDpId)
 759                     .setDestinationDpid(dstDpId)
 760                     .setTunnelType(tunType)
 761                     .build());
 762                 rpcResult = result.get();
 763                 if (!rpcResult.isSuccessful()) {
 764                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 765                             rpcResult.getErrors());
 766                 } else {
 767                     return rpcResult.getResult().getInterfaceName();
 768                 }
 769                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 770                         rpcResult.getErrors());
 771             } else {
 772                 return rpcResult.getResult().getInterfaceName();
 773             }
 774         } catch (InterruptedException | ExecutionException e) {
 775             LOG.error("getTunnelInterfaceName :Exception when getting tunnel interface Id for tunnel between {} and {}",
 776                 srcDpId, dstDpId, e);
 777         }
 778         LOG.error("getTunnelInterfaceName : Tunnel missing between dpn {}:{}", srcDpId, dstDpId);
 779         return null;
 780     }
 781
 782     // TODO Clean up the exception handling
 783     @SuppressWarnings("checkstyle:IllegalCatch")
 784     public boolean updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
 785         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
 786             .setPrimarySwitchId(naptSwitchId).build();
 787         try {
 788             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 789                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
 790         } catch (Exception ex) {
 791             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
 792                 naptSwitchId, routerName);
 793             return false;
 794         }
 795         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
 796             naptSwitchId, routerName);
 797         return true;
 798     }
 799
 800     public FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long groupId,
 801                                           long routerVpnId, int addordel) {
 802         FlowEntity flowEntity;
 803         List<MatchInfo> matches = new ArrayList<>();
 804         matches.add(MatchEthernetType.IPV4);
 805         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID));
 806
 807         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 808
 809         if (addordel == NatConstants.ADD_FLOW) {
 810             List<ActionInfo> actionsInfo = new ArrayList<>();
 811             long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 812                     idManager, routerVpnId, routerName);
 813             actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 814             LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 815             actionsInfo.add(new ActionGroup(groupId));
 816             List<InstructionInfo> instructions = new ArrayList<>();
 817             instructions.add(new InstructionApplyActions(actionsInfo));
 818
 819             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 820                     NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 821                     NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 822         } else {
 823             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 824                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 825                 NwConstants.COOKIE_SNAT_TABLE, matches, null);
 826         }
 827         return flowEntity;
 828     }
 829
 830     public FlowEntity buildSnatFlowEntityForNaptSwitch(BigInteger dpId, String routerName,
 831                                                        long routerVpnId, int addordel) {
 832         FlowEntity flowEntity;
 833         List<MatchInfo> matches = new ArrayList<>();
 834         matches.add(MatchEthernetType.IPV4);
 835         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerVpnId), MetaDataUtil.METADATA_MASK_VRFID));
 836
 837         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 838
 839         if (addordel == NatConstants.ADD_FLOW) {
 840             List<InstructionInfo> instructions = new ArrayList<>();
 841
 842             instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 843
 844             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 845                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 846                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 847         } else {
 848             flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 849                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 850                 NwConstants.COOKIE_SNAT_TABLE, matches, null);
 851         }
 852         return flowEntity;
 853     }
 854
 855     private String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 856         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 857                 .FLOWID_SEPARATOR + routerID;
 858     }
 859
 860     protected void installSnatFlows(String routerName, Long routerId, BigInteger naptSwitch, Long routerVpnId,
 861                                     TypedReadWriteTransaction<Configuration> confTx) {
 862
 863         if (routerId.equals(routerVpnId)) {
 864             LOG.debug("installSnatFlows : Installing flows for router with internalvpnId");
 865             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 866             LOG.debug("installSnatFlows : installTerminatingServiceTblEntry in naptswitch with dpnId {} for "
 867                 + "routerName {} with routerId {}", naptSwitch, routerName, routerId);
 868             externalRouterListener.installTerminatingServiceTblEntry(naptSwitch, routerName, routerId, confTx);
 869
 870             //Install default flows punting to controller in table 46(OutBoundNapt table)
 871             LOG.debug("installSnatFlows : installOutboundMissEntry in naptswitch with dpnId {} for "
 872                 + "routerName {} with routerId {}", naptSwitch, routerName, routerId);
 873             externalRouterListener.createOutboundTblEntry(naptSwitch, routerId, confTx);
 874
 875             //Table 47 point to table 21 for inbound traffic
 876             LOG.debug("installSnatFlows : installNaptPfibEntry in naptswitch with dpnId {} for router {}",
 877                 naptSwitch, routerId);
 878             externalRouterListener.installNaptPfibEntry(naptSwitch, routerId, confTx);
 879
 880             //Table 47 point to group
 881             LOG.debug("installSnatFlows : installNaptPfibExternalOutputFlow in naptswitch with dpnId {} for router {}",
 882                 naptSwitch, routerId);
 883             externalRouterListener.installNaptPfibExternalOutputFlow(routerName, routerId, naptSwitch, confTx);
 884         } else {
 885             Uuid extNetworkUuid = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 886             if (extNetworkUuid == null) {
 887                 LOG.error("onRouterAssociatedToVpn : Unable to retrieve external network Uuid for router {}",
 888                         routerName);
 889                 return;
 890             }
 891             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 892                     extNetworkUuid);
 893             if (extNwProvType == null) {
 894                 LOG.error("onRouterAssociatedToVpn : External Network Provider Type missing");
 895                 return;
 896             }
 897             //36 -> 46 ..Install flow forwarding packet to table46 from table36
 898             LOG.debug("installSnatFlows : installTerminatingServiceTblEntry in naptswitch with dpnId {} for "
 899                 + "routerName {} with BgpVpnId {}", naptSwitch, routerName, routerVpnId);
 900             externalRouterListener
 901                 .installTerminatingServiceTblEntryWithUpdatedVpnId(naptSwitch, routerName, routerId,
 902                         routerVpnId, confTx, extNwProvType);
 903
 904             //Install default flows punting to controller in table 46(OutBoundNapt table)
 905             LOG.debug("installSnatFlows : installOutboundMissEntry in naptswitch with dpnId {} for "
 906                 + "routerName {} with BgpVpnId {}", naptSwitch, routerName, routerVpnId);
 907             externalRouterListener.createOutboundTblEntryWithBgpVpn(naptSwitch, routerId, routerVpnId, confTx);
 908
 909             //Table 47 point to table 21 for inbound traffic
 910             LOG.debug("installSnatFlows : installNaptPfibEntry in naptswitch with dpnId {} for router {} "
 911                     + "with BgpVpnId {}", naptSwitch, routerId, routerVpnId);
 912             externalRouterListener.installNaptPfibEntryWithBgpVpn(naptSwitch, routerId, routerVpnId, confTx);
 913         }
 914
 915         Uuid networkId = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 916         String vpnName = getExtNetworkVpnName(routerName, networkId);
 917         if (vpnName != null) {
 918             //NAPT PFIB point to FIB table for outbound traffic
 919             long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 920             boolean shouldInstallNaptPfibWithExtNetworkVpnId = true;
 921             Collection<Uuid> externalSubnetIds = NatUtil.getExternalSubnetIdsForRouter(dataBroker, routerName);
 922             if (!externalSubnetIds.isEmpty()) {
 923                 //NAPT PFIB point to FIB table for outbound traffic - using external subnetID as vpnID.
 924                 for (Uuid externalSubnetId : externalSubnetIds) {
 925                     long externalSubnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, externalSubnetId);
 926                     if (externalSubnetVpnId != NatConstants.INVALID_ID) {
 927                         shouldInstallNaptPfibWithExtNetworkVpnId = false;
 928                         LOG.debug("installSnatFlows : installNaptPfibEntry fin naptswitch with dpnId {} for "
 929                                 + "BgpVpnId {}", naptSwitch, externalSubnetVpnId);
 930                         externalRouterListener.installNaptPfibEntry(naptSwitch, externalSubnetVpnId, confTx);
 931                     }
 932                 }
 933             }
 934             if (vpnId != NatConstants.INVALID_ID && shouldInstallNaptPfibWithExtNetworkVpnId) {
 935                 //NAPT PFIB table point to FIB table for outbound traffic - using external networkID as vpnID.
 936                 LOG.debug("installSnatFlows : installNaptPfibEntry fin naptswitch with dpnId {} for "
 937                     + "BgpVpnId {}", naptSwitch, vpnId);
 938                 externalRouterListener.installNaptPfibEntry(naptSwitch, vpnId, confTx);
 939             } else if (vpnId != NatConstants.INVALID_ID) {
 940                 LOG.debug("installSnatFlows : Associated BgpvpnId not found for router {}", routerId);
 941             }
 942
 943             //Install Fib entries for ExternalIps & program 36 -> 44
 944             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 945             String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 946             for (String externalIp : externalIps) {
 947                 removeFibEntry(rd, externalIp);
 948                 LOG.debug("installSnatFlows : advToBgpAndInstallFibAndTsFlows in naptswitch id {} "
 949                     + "with vpnName {} and externalIp {}", naptSwitch, vpnName, externalIp);
 950                 externalRouterListener.advToBgpAndInstallFibAndTsFlows(naptSwitch, NwConstants.INBOUND_NAPT_TABLE,
 951                     vpnName, routerId, routerName, externalIp, networkId, null /* external-router */, confTx);
 952                 LOG.debug("installSnatFlows : Successfully added fib entries in naptswitch {} for "
 953                     + "router {} with external IP {}", naptSwitch, routerId, externalIp);
 954             }
 955         } else {
 956             LOG.debug("installSnatFlows : Associated vpnName not found for router {}", routerId);
 957         }
 958     }
 959
 960     protected void bestEffortDeletion(long routerId, String routerName, Map<String, Long> externalIpLabel,
 961                                       TypedReadWriteTransaction<Configuration> confTx)
 962             throws ExecutionException, InterruptedException {
 963         Collection<String> newExternalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 964         if (externalIpsCache != null) {
 965             Set<String> removedExternalIps = new HashSet<>(externalIpsCache);
 966             removedExternalIps.removeAll(newExternalIps);
 967             if (removedExternalIps.isEmpty()) {
 968                 LOG.info("bestEffortDeletion : No external Ip needed to be removed in bestEffortDeletion "
 969                         + "method for router {}", routerName);
 970                 return;
 971             }
 972             Uuid networkId = NatUtil.getNetworkIdFromRouterName(dataBroker, routerName);
 973             String vpnName = getExtNetworkVpnName(routerName, networkId);
 974             if (vpnName == null) {
 975                 LOG.error("bestEffortDeletion : Vpn is not associated to externalN/w of router {}", routerName);
 976                 return;
 977             }
 978             BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 979             if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 980                 LOG.error("bestEffortDeletion : No naptSwitch is selected for router {}", routerName);
 981                 return;
 982             }
 983             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker,routerName, networkId);
 984             if (extNwProvType == null) {
 985                 return;
 986             }
 987             String gwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
 988             if (gwMacAddress != null) {
 989                 LOG.debug("bestEffortDeletion : External Gateway MAC address {} found for External Router ID {}",
 990                         gwMacAddress, routerId);
 991             } else {
 992                 LOG.error("bestEffortDeletion : No External Gateway MAC address found for External Router ID {}",
 993                         routerId);
 994                 return;
 995             }
 996             if (extNwProvType == ProviderTypes.VXLAN) {
 997                 for (String externalIp : removedExternalIps) {
 998                     externalRouterListener.clearBgpRoutes(externalIp, vpnName);
 999                     externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerName, routerId, externalIp,
1000                         vpnName, networkId, NatConstants.DEFAULT_LABEL_VALUE, gwMacAddress, true, confTx);
1001                     LOG.debug("bestEffortDeletion : Successfully removed fib entry for externalIp {} for routerId {} "
1002                                     + "on NAPT switch {} ", externalIp, routerId, naptSwitch);
1003                 }
1004             } else {
1005                 if (externalIpLabel == null || externalIpLabel.isEmpty()) {
1006                     LOG.error("bestEffortDeletion : ExternalIpLabel map is empty for router {}", routerName);
1007                     return;
1008                 }
1009                 Long label;
1010                 for (String externalIp : removedExternalIps) {
1011                     if (externalIpLabel.containsKey(externalIp)) {
1012                         label = externalIpLabel.get(externalIp);
1013                         LOG.debug("bestEffortDeletion : Label {} for ExternalIp {} for router {}",
1014                                 label, externalIp, routerName);
1015                     } else {
1016                         LOG.debug("bestEffortDeletion : Label for ExternalIp {} is not found for router {}",
1017                                 externalIp, routerName);
1018                         continue;
1019                     }
1020                     externalRouterListener.clearBgpRoutes(externalIp, vpnName);
1021                     externalRouterListener.delFibTsAndReverseTraffic(naptSwitch, routerName, routerId, externalIp,
1022                             vpnName, networkId, label, gwMacAddress, true, confTx);
1023                     LOG.debug("bestEffortDeletion : Successfully removed fib entries in switch {} for router {} "
1024                             + "and externalIps {}", naptSwitch, routerId, externalIp);
1025                 }
1026             }
1027         } else {
1028             LOG.debug("bestEffortDeletion : No external IP found for router {}", routerId);
1029         }
1030     }
1031
1032     private void removeFibEntry(String rd, String prefix) {
1033         InstanceIdentifier.InstanceIdentifierBuilder<VrfEntry> idBuilder =
1034             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd))
1035                 .child(VrfEntry.class, new VrfEntryKey(prefix));
1036         InstanceIdentifier<VrfEntry> vrfEntryId = idBuilder.build();
1037         Optional<VrfEntry> ent = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId);
1038         if (ent.isPresent()) {
1039             LOG.debug("removeFibEntry : Removing Fib entry rd {} prefix {}", rd, prefix);
1040             fibManager.removeFibEntry(rd, prefix, null);
1041         }
1042     }
1043
1044     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
1045         externalRouterListener.subnetRegisterMapping(routerEntry, segmentId);
1046     }
1047 }