natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 145 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 146 import org.opendaylight.yangtools.yang.common.RpcResult;
 147 import org.slf4j.Logger;
 148 import org.slf4j.LoggerFactory;
 149
 150
 151 @Singleton
 152 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 153     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 154
 155     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 156     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 157
 158     private final DataBroker dataBroker;
 159     private final ManagedNewTransactionRunner txRunner;
 160     private final IMdsalApiManager mdsalManager;
 161     private final ItmRpcService itmManager;
 162     private final OdlInterfaceRpcService odlInterfaceRpcService;
 163     private final IdManagerService idManager;
 164     private final NaptManager naptManager;
 165     private final NAPTSwitchSelector naptSwitchSelector;
 166     private final IBgpManager bgpManager;
 167     private final VpnRpcService vpnService;
 168     private final FibRpcService fibService;
 169     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 170     private final NaptEventHandler naptEventHandler;
 171     private final NaptPacketInHandler naptPacketInHandler;
 172     private final IFibManager fibManager;
 173     private final IVpnManager vpnManager;
 174     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 175     private final NatMode natMode;
 176     private final IElanService elanManager;
 177     private final JobCoordinator coordinator;
 178     private final IInterfaceManager interfaceManager;
 179     private final NatOverVxlanUtil natOverVxlanUtil;
 180     private final int snatPuntTimeout;
 181
 182     @Inject
 183     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 184                                    final ItmRpcService itmManager,
 185                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 186                                    final IdManagerService idManager,
 187                                    final NaptManager naptManager,
 188                                    final NAPTSwitchSelector naptSwitchSelector,
 189                                    final IBgpManager bgpManager,
 190                                    final VpnRpcService vpnService,
 191                                    final FibRpcService fibService,
 192                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 193                                    final NaptEventHandler naptEventHandler,
 194                                    final NaptPacketInHandler naptPacketInHandler,
 195                                    final IFibManager fibManager,
 196                                    final IVpnManager vpnManager,
 197                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 198                                    final NatserviceConfig config,
 199                                    final IElanService elanManager,
 200                                    final JobCoordinator coordinator,
 201                                    final NatOverVxlanUtil natOverVxlanUtil,
 202                                    final IInterfaceManager interfaceManager) {
 203         super(Routers.class, ExternalRoutersListener.class);
 204         this.dataBroker = dataBroker;
 205         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 206         this.mdsalManager = mdsalManager;
 207         this.itmManager = itmManager;
 208         this.odlInterfaceRpcService = odlInterfaceRpcService;
 209         this.idManager = idManager;
 210         this.naptManager = naptManager;
 211         this.naptSwitchSelector = naptSwitchSelector;
 212         this.bgpManager = bgpManager;
 213         this.vpnService = vpnService;
 214         this.fibService = fibService;
 215         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 216         this.naptEventHandler = naptEventHandler;
 217         this.naptPacketInHandler = naptPacketInHandler;
 218         this.fibManager = fibManager;
 219         this.vpnManager = vpnManager;
 220         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 221         this.elanManager = elanManager;
 222         this.coordinator = coordinator;
 223         this.interfaceManager = interfaceManager;
 224         this.natOverVxlanUtil = natOverVxlanUtil;
 225         if (config != null) {
 226             this.natMode = config.getNatMode();
 227             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 228         } else {
 229             this.natMode = NatMode.Controller;
 230             this.snatPuntTimeout = 0;
 231         }
 232     }
 233
 234     @Override
 235     @PostConstruct
 236     public void init() {
 237         LOG.info("{} init", getClass().getSimpleName());
 238         // This class handles ExternalRouters for Controller SNAT mode.
 239         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 240         if (natMode == NatMode.Controller) {
 241             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 242             NatUtil.createGroupIdPool(idManager);
 243         }
 244     }
 245
 246     @Override
 247     protected InstanceIdentifier<Routers> getWildCardPath() {
 248         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 249     }
 250
 251     @Override
 252     // TODO Clean up the exception handling
 253     @SuppressWarnings("checkstyle:IllegalCatch")
 254     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 255         // Populate the router-id-name container
 256         String routerName = routers.getRouterName();
 257         LOG.info("add : external router event for {}", routerName);
 258         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 259         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 260         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 261         try {
 262             if (routers.isEnableSnat()) {
 263                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 264                     () -> Collections.singletonList(
 265                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 266                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 267                             long bgpVpnId = NatConstants.INVALID_ID;
 268                             if (bgpVpnUuid != null) {
 269                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 270                             }
 271                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 272                             // Allocate Primary Napt Switch for this router
 273                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 274                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 275                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 276                             }
 277                         }
 278                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 279             } else {
 280                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 281             }
 282         } catch (Exception ex) {
 283             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 284                     routerName, ex);
 285         }
 286     }
 287
 288     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 289                                  TypedWriteTransaction<Configuration> confTx) {
 290         String routerName = routers.getRouterName();
 291         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 292
 293         naptManager.initialiseExternalCounter(routers, routerId);
 294         subnetRegisterMapping(routers, routerId);
 295
 296         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 297             primarySwitchId, routerName);
 298
 299         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 300                 routers.getNetworkId());
 301         if (extNwProvType == null) {
 302             LOG.error("handleEnableSnat : External Network Provider Type missing");
 303             return;
 304         }
 305
 306         if (bgpVpnId != NatConstants.INVALID_ID) {
 307             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 308                     extNwProvType);
 309         } else {
 310             // write metadata and punt
 311             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 312             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, confTx);
 313             // Now install entries in SNAT tables to point to Primary for each router
 314             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 315             for (BigInteger dpnId : switches) {
 316                 // Handle switches and NAPT switches separately
 317                 if (!dpnId.equals(primarySwitchId)) {
 318                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 319                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 320                 }
 321             }
 322         }
 323
 324         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 325         if (externalIps.isEmpty()) {
 326             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 327             return;
 328         } else {
 329             for (String externalIpAddrPrefix : externalIps) {
 330                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 331                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 332                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 333                 );
 334             }
 335         }
 336         LOG.debug("handleEnableSnat : Exit");
 337     }
 338
 339     private BigInteger getPrimaryNaptSwitch(String routerName) {
 340         // Allocate Primary Napt Switch for this router
 341         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 342         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 343             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 344                 primarySwitchId, routerName);
 345             return primarySwitchId;
 346         }
 347         return selectNewNAPTSwitch(routerName);
 348     }
 349
 350     private BigInteger selectNewNAPTSwitch(String routerName) {
 351         // Allocated an id from VNI pool for the Router.
 352         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 353         BigInteger primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 354         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 355
 356         return primarySwitchId;
 357     }
 358
 359     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 360                                                      TypedWriteTransaction<Configuration> confTx) {
 361         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 362         if (extVpnId == NatConstants.INVALID_ID) {
 363             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 364             extVpnId = routerId;
 365         }
 366         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 367         if (externalIps.isEmpty()) {
 368             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 369                 dpnId, extVpnId);
 370             return;
 371         }
 372         for (String ip : externalIps) {
 373             Uuid subnetId = getSubnetIdForFixedIp(ip);
 374             if (subnetId != null) {
 375                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 376                 if (subnetVpnId != NatConstants.INVALID_ID) {
 377                     extVpnId = subnetVpnId;
 378                 }
 379                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 380                     dpnId, extVpnId, subnetId);
 381                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 382                 if (postNaptFlowEntity != null) {
 383                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 384                 }
 385             }
 386         }
 387     }
 388
 389     @Nullable
 390     private Uuid getSubnetIdForFixedIp(String ip) {
 391         if (ip != null) {
 392             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 393             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 394             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 395         }
 396         LOG.error("getSubnetIdForFixedIp : ip is null");
 397         return null;
 398     }
 399
 400     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 401         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 402         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 403         int counter = 0;
 404         int extIpCounter = externalIps.size();
 405         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 406                 counter, extIpCounter);
 407         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 408         if (subnetIds == null) {
 409             return;
 410         }
 411         for (Uuid subnet : subnetIds) {
 412             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 413             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 414                 .builder(Subnetmaps.class)
 415                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 416                 .build();
 417             Optional<Subnetmap> sn;
 418             try {
 419                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 420                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 421             } catch (ReadFailedException e) {
 422                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 423                 sn = Optional.absent();
 424             }
 425             if (sn.isPresent()) {
 426                 // subnets
 427                 Subnetmap subnetmapEntry = sn.get();
 428                 String subnetString = subnetmapEntry.getSubnetIp();
 429                 String[] subnetSplit = subnetString.split("/");
 430                 String subnetIp = subnetSplit[0];
 431                 try {
 432                     InetAddress address = InetAddress.getByName(subnetIp);
 433                     if (address instanceof Inet6Address) {
 434                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 435                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 436                         continue;
 437                     }
 438                 } catch (UnknownHostException e) {
 439                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 440                     return;
 441                 }
 442                 String subnetPrefix = "0";
 443                 if (subnetSplit.length == 2) {
 444                     subnetPrefix = subnetSplit[1];
 445                 }
 446                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 447                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 448                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 449                 //externalIps
 450                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 451                         counter, extIpCounter);
 452                 if (extIpCounter != 0) {
 453                     if (counter < extIpCounter) {
 454                         String[] ipSplit = externalIps.get(counter).split("/");
 455                         String externalIp = ipSplit[0];
 456                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 457                         if (ipSplit.length == 2) {
 458                             extPrefix = ipSplit[1];
 459                         }
 460                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 461                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 462                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 463                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 464                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 465                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 466                     } else {
 467                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 468                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 469                         String[] ipSplit = externalIps.get(counter).split("/");
 470                         String externalIp = ipSplit[0];
 471                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 472                         if (ipSplit.length == 2) {
 473                             extPrefix = ipSplit[1];
 474                         }
 475                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 476                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 477                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 478                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 479                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 480                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 481                             externalIp, extPrefix);
 482                     }
 483                 }
 484                 counter++;
 485                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 486             } else {
 487                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 488             }
 489         }
 490     }
 491
 492     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 493             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 494             throws ExecutionException, InterruptedException {
 495         //Check if BGP VPN exists. If exists then invoke the new method.
 496         if (bgpVpnId != NatConstants.INVALID_ID) {
 497             if (bgpVpnUuid != null) {
 498                 String bgpVpnName = bgpVpnUuid.getValue();
 499                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 500                     bgpVpnId, bgpVpnName);
 501                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 502                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 503                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 504             }
 505             if (create) {
 506                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 507             } else {
 508                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 509             }
 510             return;
 511         }
 512
 513         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 514         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 515     }
 516
 517     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 518             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 519         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 520         if (switches.isEmpty()) {
 521             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 522             return;
 523         }
 524         if (routerId == NatConstants.INVALID_ID) {
 525             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 526                     + "default NAT route in FIB", routerName);
 527             return;
 528         }
 529         for (BigInteger dpnId : switches) {
 530             if (create) {
 531                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 532                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 533                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 534             } else {
 535                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 536                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 537                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 538             }
 539         }
 540     }
 541
 542     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 543         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 544         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 545         if (dpnIds.isEmpty()) {
 546             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 547                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 548             return;
 549         }
 550         for (BigInteger dpnId : dpnIds) {
 551             if (bgpVpnId != NatConstants.INVALID_ID) {
 552                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 553                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 554                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 555             } else {
 556                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 557                     + "in dpn {} for the internal vpn", routerId, dpnId);
 558                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 559             }
 560         }
 561     }
 562
 563     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 564             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 565             throws ExecutionException, InterruptedException {
 566         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 567         if (dpnIds.isEmpty()) {
 568             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 569                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 570             return;
 571         }
 572         for (BigInteger dpnId : dpnIds) {
 573             if (bgpVpnId != NatConstants.INVALID_ID) {
 574                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 575                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 576                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 577             } else {
 578                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 579                     + "in dpn {} for the internal vpn", routerId, dpnId);
 580                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 581             }
 582         }
 583     }
 584
 585     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 586         TypedWriteTransaction<Configuration> confTx) {
 587         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 588         if (routerId != NatConstants.INVALID_ID) {
 589             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 590                     primarySwitchId, routerId);
 591             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 592         } else {
 593             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 594                 + "createAndInstallMissEntry", routerName);
 595         }
 596     }
 597
 598     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 599         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 600                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 601     }
 602
 603     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 604         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 605                 .FLOWID_SEPARATOR + vpnId;
 606     }
 607
 608     public BigInteger getCookieOutboundFlow(long routerId) {
 609         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 610             BigInteger.valueOf(routerId));
 611     }
 612
 613     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 614         long l4SrcPortField;
 615         long l4DstPortField;
 616         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 617
 618         if (protocol == NwConstants.IP_PROT_TCP) {
 619             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 620             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 621         } else {
 622             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 623             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 624         }
 625         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 626                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 627                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 628                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 629                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 630                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 631                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 632                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 633                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 634                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 635                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 636                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 637                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 638                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 639                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 640
 641         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 642                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 643     }
 644
 645     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 646         List<MatchInfo> matches = new ArrayList<>();
 647         matches.add(MatchEthernetType.IPV4);
 648         matches.add(MatchIpProtocol.ICMP);
 649         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 650
 651         List<ActionInfo> actionInfos = new ArrayList<>();
 652         actionInfos.add(new ActionDrop());
 653
 654         List<InstructionInfo> instructions = new ArrayList<>();
 655         instructions.add(new InstructionApplyActions(actionInfos));
 656
 657         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 658                 NwConstants.IP_PROT_ICMP);
 659         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 660                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 661                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 662         return flow;
 663     }
 664
 665     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 666         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 667         BigInteger cookie = getCookieOutboundFlow(routerId);
 668         List<MatchInfo> matches = new ArrayList<>();
 669         matches.add(MatchEthernetType.IPV4);
 670         matches.add(new MatchIpProtocol((short)protocol));
 671         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 672
 673         List<InstructionInfo> instructions = new ArrayList<>();
 674         List<ActionInfo> actionsInfos = new ArrayList<>();
 675         actionsInfos.add(new ActionPuntToController());
 676         if (snatPuntTimeout != 0) {
 677             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 678         }
 679         instructions.add(new InstructionApplyActions(actionsInfos));
 680
 681         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 682         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 683             5, flowRef, 0, 0,
 684             cookie, matches, instructions);
 685         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 686         return flowEntity;
 687     }
 688
 689     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 690         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 691         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 692         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 693         mdsalManager.addFlow(confTx, tcpflowEntity);
 694
 695         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 696         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 697         mdsalManager.addFlow(confTx, udpflowEntity);
 698
 699         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 700         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 701         mdsalManager.addFlow(confTx, icmpDropFlow);
 702     }
 703
 704     @Nullable
 705     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 706         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 707         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 708         try {
 709             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 710                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 711                     .setSourceDpid(srcDpId)
 712                     .setDestinationDpid(dstDpId)
 713                     .setTunnelType(tunType)
 714                     .build());
 715             rpcResult = result.get();
 716             if (!rpcResult.isSuccessful()) {
 717                 tunType = TunnelTypeGre.class;
 718                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 719                     .setSourceDpid(srcDpId)
 720                     .setDestinationDpid(dstDpId)
 721                     .setTunnelType(tunType)
 722                     .build());
 723                 rpcResult = result.get();
 724                 if (!rpcResult.isSuccessful()) {
 725                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 726                             rpcResult.getErrors());
 727                 } else {
 728                     return rpcResult.getResult().getInterfaceName();
 729                 }
 730                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 731                         rpcResult.getErrors());
 732             } else {
 733                 return rpcResult.getResult().getInterfaceName();
 734             }
 735         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 736             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 737                     + "between {} and {}", srcDpId, dstDpId, e);
 738         }
 739
 740         return null;
 741     }
 742
 743     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 744         TypedWriteTransaction<Configuration> confTx) {
 745
 746         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 747         // Install miss entry pointing to group
 748         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 749         mdsalManager.addFlow(confTx, flowEntity);
 750     }
 751
 752     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 753             String routerName, long routerId) {
 754         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 755             dpnId, bucketInfo.get(0));
 756         // Install the select group
 757         long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 758             NatUtil.getGroupIdKey(routerName));
 759         if (groupId == NatConstants.INVALID_ID) {
 760             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 761             return;
 762         }
 763         GroupEntity groupEntity =
 764             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 765         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 766         mdsalManager.syncInstallGroup(groupEntity);
 767         // Install miss entry pointing to group
 768         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 769         if (flowEntity == null) {
 770             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 771                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 772                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 773             return;
 774         }
 775         mdsalManager.installFlow(flowEntity);
 776     }
 777
 778     void installGroup(BigInteger dpnId, String routerName, long groupId, List<BucketInfo> bucketInfo) {
 779         GroupEntity groupEntity =
 780             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 781         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 782         mdsalManager.syncInstallGroup(groupEntity);
 783     }
 784
 785     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 786         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 787                 dpId, routerName, groupId);
 788         List<MatchInfo> matches = new ArrayList<>();
 789         matches.add(MatchEthernetType.IPV4);
 790         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 791
 792         List<ActionInfo> actionsInfo = new ArrayList<>();
 793         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 794                 routerId, routerName);
 795         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 796         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 797         actionsInfo.add(new ActionGroup(groupId));
 798         List<InstructionInfo> instructions = new ArrayList<>();
 799         instructions.add(new InstructionApplyActions(actionsInfo));
 800         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 801         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 802                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 803                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 804
 805         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 806         return flowEntity;
 807     }
 808
 809     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 810
 811         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 812             routerName);
 813         List<MatchInfo> matches = new ArrayList<>();
 814         matches.add(MatchEthernetType.IPV4);
 815         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 816
 817         List<InstructionInfo> instructions = new ArrayList<>();
 818         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 819
 820         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 821         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 822             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 823             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 824
 825         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 826         return flowEntity;
 827     }
 828
 829     // TODO : Replace this with ITM Rpc once its available with full functionality
 830     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 831         TypedWriteTransaction<Configuration> confTx) {
 832
 833         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 834             dpnId, routerName);
 835         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 836         if (flowEntity == null) {
 837             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 838                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 839                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 840                     dpnId, routerName);
 841             return;
 842         }
 843         mdsalManager.addFlow(confTx, flowEntity);
 844
 845     }
 846
 847     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 848         List<MatchInfo> matches = new ArrayList<>();
 849         matches.add(MatchEthernetType.IPV4);
 850         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 851                 idManager, routerId, routerName);
 852         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 853         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 854         List<InstructionInfo> instructions = new ArrayList<>();
 855         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 856                 MetaDataUtil.METADATA_MASK_VRFID));
 857         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 858         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 859                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 860                 instructions);
 861         return flowEntity;
 862     }
 863
 864     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 865         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 866                 .FLOWID_SEPARATOR + routerID;
 867     }
 868
 869     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 870         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 871                 .FLOWID_SEPARATOR + routerID;
 872     }
 873
 874     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 875         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 876         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 877         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 878         List<BucketInfo> listBucketInfo = new ArrayList<>();
 879
 880         if (ifNamePrimary != null) {
 881             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 882             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 883                     interfaceManager, ifNamePrimary, routerId, true);
 884             if (listActionInfoPrimary.isEmpty()) {
 885                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 886                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 887                         ifNamePrimary);
 888             }
 889         } else {
 890             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 891                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 892         }
 893         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 894
 895         listBucketInfo.add(0, bucketPrimary);
 896         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 897     }
 898
 899     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 900                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 901         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 902         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 903         List<BucketInfo> listBucketInfo = new ArrayList<>();
 904
 905         if (ifNamePrimary != null) {
 906             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 907                     ifNamePrimary);
 908             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 909                     interfaceManager, ifNamePrimary, routerId, true);
 910             if (listActionInfoPrimary.isEmpty()) {
 911                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 912                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 913                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 914             }
 915         } else {
 916             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 917                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 918         }
 919         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 920
 921         listBucketInfo.add(0, bucketPrimary);
 922         return listBucketInfo;
 923     }
 924
 925     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 926         TypedWriteTransaction<Configuration> confTx) {
 927
 928        /*
 929         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 930         */
 931
 932         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 933
 934 /*
 935         List<BucketInfo> listBucketInfo = new ArrayList<>();
 936         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 937         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 938         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 939         listBucketInfo.add(0, bucketPrimary);
 940 */
 941
 942         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 943         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 944         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 945         installNaptPfibEntry(dpnId, routerId, confTx);
 946         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 947         if (networkId != null) {
 948             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 949             if (vpnUuid != null) {
 950                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 951                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 952                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 953                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 954                     if (vpnId != NatConstants.INVALID_ID) {
 955                         installNaptPfibEntry(dpnId, vpnId, null);
 956                     }
 957                     return Collections.emptyList();
 958                 });
 959             } else {
 960                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 961             }
 962         } else {
 963             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 964         }
 965     }
 966
 967     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
 968             @Nullable TypedWriteTransaction<Configuration> confTx) {
 969         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 970         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 971         if (confTx != null) {
 972             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 973         } else {
 974             mdsalManager.installFlow(naptPfibFlowEntity);
 975         }
 976     }
 977
 978     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 979
 980         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 981         List<MatchInfo> matches = new ArrayList<>();
 982         matches.add(MatchEthernetType.IPV4);
 983         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 984
 985         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 986         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 987         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
 988         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 989         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 990
 991         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
 992         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
 993             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 994             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
 995         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
 996         return flowEntity;
 997     }
 998
 999     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1000         long routerId, String routerName, String externalIp) {
1001         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1002             dpnId, routerId, externalIp);
1003         Uuid networkId = router.getNetworkId();
1004         if (networkId == null) {
1005             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1006             return;
1007         }
1008         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1009         if (vpnName == null) {
1010             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1011                     + "configuration {} in router {}", networkId, externalIp, routerId);
1012             return;
1013         }
1014         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1015             externalIp, networkId, router, confTx);
1016         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1017             dpnId, routerId, externalIp);
1018     }
1019
1020     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1021                                                 final long routerId, final String routerName, final String externalIp,
1022                                                 final Uuid extNetworkId, @Nullable final Routers router,
1023                                                 final TypedWriteTransaction<Configuration> confTx) {
1024         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1025                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1026         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1027         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1028         if (rd == null || rd.isEmpty()) {
1029             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1030             return;
1031         }
1032         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1033         if (extNwProvType == null) {
1034             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1035             return;
1036         }
1037         if (extNwProvType == ProviderTypes.VXLAN) {
1038             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1039                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1040             return;
1041         }
1042         //Generate VPN label for the external IP
1043         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1044             .setIpPrefix(externalIp).build();
1045         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1046
1047         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1048         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1049             if (result.isSuccessful()) {
1050                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1051                 GenerateVpnLabelOutput output = result.getResult();
1052                 final long label = output.getLabel();
1053
1054                 int externalIpInDsFlag = 0;
1055                 //Get IPMaps from the DB for the router ID
1056                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1057                 for (IpMap dbIpMap : dbIpMaps) {
1058                     String dbExternalIp = dbIpMap.getExternalIp();
1059                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1060                     if (dbExternalIp.contains(externalIp)) {
1061                         String dbInternalIp = dbIpMap.getInternalIp();
1062                         IpMapKey dbIpMapKey = dbIpMap.key();
1063                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1064                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1065                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1066                             .setExternalIp(dbExternalIp).setLabel(label).build();
1067                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1068                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1069                         externalIpInDsFlag++;
1070                     }
1071                 }
1072                 if (externalIpInDsFlag <= 0) {
1073                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1074                             + "Failed to update label {} for routerId {} in DS",
1075                             externalIp, label, routerId);
1076                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1077                         + " found in DS for router %s", label, externalIp, routerId);
1078                     return Futures.immediateFailedFuture(new Exception(errMsg));
1079                 }
1080                 //Inform BGP
1081                 long l3vni = 0;
1082                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1083                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1084                 }
1085                 Routers extRouter = router != null ? router :
1086                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1087                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1088                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1089                     RouteOrigin.STATIC, dpnId);
1090
1091                 //Install custom FIB routes
1092                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1093                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1094                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1095                         extNwProvType);
1096                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1097
1098                 //Install custom FIB routes - FIB table.
1099                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1100                         routerName, externalIp);
1101                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1102                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1103                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1104                 }
1105                 String externalVpn = vpnName;
1106                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1107                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1108                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1109                             externalSubnetId);
1110                     if (externalSubnet.isPresent()) {
1111                         externalVpn =  externalSubnetId.getValue();
1112                     }
1113                 }
1114                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1115                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1116                     .setVpnName(externalVpn)
1117                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1118                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1119                     .setInstruction(fibTableCustomInstructions).build();
1120                 return fibService.createFibEntry(input);
1121             } else {
1122                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1123                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1124                     externalIp, vpnName, result.getErrors());
1125                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1126             }
1127         }, MoreExecutors.directExecutor());
1128
1129         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1130
1131             @Override
1132             public void onFailure(@NonNull Throwable error) {
1133                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1134             }
1135
1136             @Override
1137             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1138                 if (result.isSuccessful()) {
1139                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1140                             externalIp);
1141                 } else {
1142                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1143                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1144                 }
1145             }
1146         }, MoreExecutors.directExecutor());
1147     }
1148
1149     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1150             String externalIp) {
1151         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1152         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1153         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1154                 externalIp, router);
1155         int instructionIndex = 0;
1156         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1157             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1158             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1159                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1160             instructionIndex++;
1161         }
1162
1163         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1164         return fibTableCustomInstructions;
1165     }
1166
1167     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1168         TypedWriteTransaction<Configuration> confTx) {
1169         List<MatchInfo> matches = new ArrayList<>();
1170         matches.add(MatchEthernetType.MPLS_UNICAST);
1171         matches.add(new MatchMplsLabel(serviceId));
1172
1173         List<Instruction> instructions = new ArrayList<>();
1174         List<ActionInfo> actionsInfos = new ArrayList<>();
1175         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1176         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1177         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1178         instructions.add(writeInstruction);
1179         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1180
1181         // Install the flow entry in L3_LFIB_TABLE
1182         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1183
1184         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1185             10, flowRef, 0, 0,
1186             COOKIE_VM_LFIB_TABLE, matches, instructions);
1187
1188         mdsalManager.addFlow(confTx, dpId, flowEntity);
1189
1190         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1191     }
1192
1193     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1194         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1195         ProviderTypes extNwProvType) {
1196
1197         List<MatchInfo> mkMatches = new ArrayList<>();
1198
1199         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1200             dpnId, serviceId, customInstructions);
1201
1202         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1203             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1204         } else {
1205             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1206         }
1207
1208         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1209             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1210                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1211                 String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
1212                 COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1213
1214         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1215     }
1216
1217     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1218         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1219             new RouterIdsKey(routerId)).build();
1220     }
1221
1222     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1223         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1224                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1225     }
1226
1227     @Override
1228     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1229         LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1230         String routerName = original.getRouterName();
1231         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1232         if (routerId == NatConstants.INVALID_ID) {
1233             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1234             return;
1235         }
1236         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1237             List<ListenableFuture<Void>> futures = new ArrayList<>();
1238             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1239                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1240                     long bgpVpnId = NatConstants.INVALID_ID;
1241                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1242                     if (bgpVpnUuid != null) {
1243                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1244                     }
1245                     //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1246                     /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1247                      * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1248                      */
1249                     BigInteger dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1250                     boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(BigInteger.ZERO));
1251                     Uuid networkId = original.getNetworkId();
1252                     // Check if its update on SNAT flag
1253                     boolean originalSNATEnabled = original.isEnableSnat();
1254                     boolean updatedSNATEnabled = update.isEnableSnat();
1255                     LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1256                             + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1257                         originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1258                     // Cluster Reboot Case Handling
1259                     // 1. DPN not elected during add event(due to none of the OVS connected)
1260                     // 2. Update event called with changes of parameters(but enableSnat is not changed)
1261                     // 3. First Elect dpnId and process other changes with valid dpnId
1262                     if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1263                         if (originalSNATEnabled && !updatedSNATEnabled) {
1264                             if (isPrimaryNaptSwitchNotSelected) {
1265                                 LOG.info("No Action to be taken when SNAT is disabled "
1266                                     + "with no Napt Switch Election for Router {}", routerName);
1267                                 return;
1268                             }
1269                             //SNAT disabled for the router
1270                             Uuid networkUuid = original.getNetworkId();
1271                             LOG.info("update : SNAT disabled for Router {}", routerName);
1272                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1273                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1274                                     removeFlowInvTx);
1275                         }  else if (updatedSNATEnabled) {
1276                             LOG.info("update : SNAT enabled for Router {}", routerName);
1277                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1278                                 true, writeFlowInvTx);
1279                             if (isPrimaryNaptSwitchNotSelected) {
1280                                 dpnId = selectNewNAPTSwitch(routerName);
1281                                 if (dpnId != null && !dpnId.equals(BigInteger.ZERO)) {
1282                                     handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1283                                 } else {
1284                                     LOG.error("update : Failed to elect Napt Switch During update event"
1285                                         + " of router {}", routerName);
1286                                 }
1287                             }
1288                         }
1289                         LOG.info("update : no need to process external/subnet changes as it's will taken care"
1290                             + "in handleDisableSnat/handleEnableSnat");
1291                         return;
1292                     }
1293                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1294                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1295                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1296                     }
1297
1298                     if (updatedSNATEnabled != originalSNATEnabled) {
1299                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1300                                 + "handleDisableSnat/handleEnableSnat");
1301                         return;
1302                     }
1303                     //Check if the Update is on External IPs
1304                     LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1305                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1306                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1307
1308                     //Check if the External IPs are removed during the update.
1309                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1310                     removedExternalIps.removeAll(updatedExternalIps);
1311                     if (removedExternalIps.size() > 0) {
1312                         LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1313                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1314                                 removedExternalIps, original.getExtGwMacAddress(),
1315                                 dpnId, networkId);
1316
1317                         for (String removedExternalIp : removedExternalIps) {
1318                 /*
1319                     1) Remove the mappings in the IntExt IP model which has external IP.
1320                     2) Remove the external IP in the ExternalCounter model.
1321                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1322                        least loaded external IP.
1323                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1324                     4) Increase the count of the allocated external IP by one.
1325                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1326                      i.e. the route for the external IP is absent.
1327                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1328                      the removed external IPs and also from the model.
1329                     7) Advertise to the BGP for removing the route for the removed external IPs.
1330                  */
1331
1332                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1333                             String externalIp = externalIpParts[0];
1334                             String externalIpPrefix = externalIpParts[1];
1335                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1336
1337                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1338                                     + "entries for removed external IP {}", externalIpAddrStr);
1339                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1340                             String vpnName = "";
1341                             if (vpnUuId != null) {
1342                                 vpnName = vpnUuId.getValue();
1343                             }
1344                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1345                                     update.getExtGwMacAddress(), removeFlowInvTx);
1346
1347                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1348                             //Get the internal IPs which are associated to the removed external IPs
1349                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1350                             List<String> removedInternalIps = new ArrayList<>();
1351                             for (IpMap ipMap : ipMaps) {
1352                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1353                                     removedInternalIps.add(ipMap.getInternalIp());
1354                                 }
1355                             }
1356
1357                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1358                             for (String removedInternalIp : removedInternalIps) {
1359                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1360                                                 + "router ID {} from the IntExtIP model",
1361                                         removedInternalIp, routerId);
1362                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1363                             }
1364
1365                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1366                                             + "router ID {} from the ExternalIpsCounter model.",
1367                                     externalIpAddrStr, routerId);
1368                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1369
1370                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1371                                     + "whose external IPs were removed.");
1372                             for (String removedInternalIp : removedInternalIps) {
1373                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1374                                         removedInternalIp, writeFlowInvTx);
1375                             }
1376                             LOG.debug("update : Remove the NAPT translation entries from "
1377                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1378                             //Get the internalIP and internal Port which were associated to the removed external IP.
1379                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1380                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1381                                     .builder(IntextIpPortMap.class)
1382                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1383                             Optional<IpPortMapping> ipPortMapping;
1384                             try {
1385                                 ipPortMapping = SingleTransactionDataBroker
1386                                             .syncReadOptional(dataBroker,
1387                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1388                             } catch (ReadFailedException e) {
1389                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1390                                 ipPortMapping = Optional.absent();
1391                             }
1392
1393                             if (ipPortMapping.isPresent()) {
1394                                 for (IntextIpProtocolType intextIpProtocolType :
1395                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1396                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1397                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1398                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1399                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1400                                             List<String> removedInternalIpPorts =
1401                                                     protoTypesIntIpPortsMap.get(protoType);
1402                                             if (removedInternalIpPorts != null) {
1403                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1404                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1405                                             } else {
1406                                                 removedInternalIpPorts = new ArrayList<>();
1407                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1408                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1409                                             }
1410                                         }
1411                                     }
1412                                 }
1413                             }
1414
1415                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1416                             // the removed external IP.
1417                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1418                                     protoTypesIntIpPortsMap.entrySet();
1419                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1420                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1421                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1422                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1423                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1424                                     // Remove the IP port map from the intext-ip-port-map model,
1425                                     // which were containing the removed external IP
1426                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1427                                             protocolType);
1428                                     //Remove the IP port incomint packer map.
1429                                     naptPacketInHandler.removeIncomingPacketMap(
1430                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1431                                     String[] removedInternalIpPortParts = removedInternalIpPort
1432                                             .split(NatConstants.COLON_SEPARATOR);
1433                                     if (removedInternalIpPortParts.length == 2) {
1434                                         String removedInternalIp = removedInternalIpPortParts[0];
1435                                         String removedInternalPort = removedInternalIpPortParts[1];
1436                                         List<String> removedInternalPortsList =
1437                                                 internalIpPortMap.get(removedInternalPort);
1438                                         if (removedInternalPortsList != null) {
1439                                             removedInternalPortsList.add(removedInternalPort);
1440                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1441                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1442                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1443                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1444                                             //Remove the NAPT translation entries from Outbound NAPT table
1445                                             naptEventHandler.removeNatFlows(dpnId,
1446                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1447                                                 routerId, removedInternalIp,
1448                                                 Integer.parseInt(removedInternalPort),
1449                                                 protocolType.getName());
1450                                             naptEventHandler.removeNatFlows(dpnId,
1451                                                 NwConstants.INBOUND_NAPT_TABLE,
1452                                                 routerId, removedInternalIp,
1453                                                 Integer.parseInt(removedInternalPort),
1454                                                 protocolType.getName());
1455                                         } else {
1456                                             removedInternalPortsList = new ArrayList<>();
1457                                             removedInternalPortsList.add(removedInternalPort);
1458                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1459                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1460                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1461                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1462                                             //Remove the NAPT translation entries from Outbound NAPT table
1463                                             naptEventHandler.removeNatFlows(dpnId,
1464                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1465                                                 routerId, removedInternalIp,
1466                                                 Integer.parseInt(removedInternalPort),
1467                                                 protocolType.getName());
1468                                             naptEventHandler.removeNatFlows(dpnId,
1469                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1470                                                 Integer.parseInt(removedInternalPort),
1471                                                 protocolType.getName());
1472                                         }
1473                                     }
1474                                 }
1475                             }
1476
1477                             // Delete the entry from SnatIntIpPortMap DS
1478                             Set<String> internalIps = internalIpPortMap.keySet();
1479                             for (String internalIp : internalIps) {
1480                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1481                                         + "model SnatIntIpPortMap", internalIp);
1482                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1483                             }
1484
1485                             naptManager.removeNaptPortPool(externalIp);
1486                         }
1487                         LOG.debug(
1488                                 "update : End processing of the External IPs removal for router {}", routerName);
1489                     }
1490
1491                     //Check if the External IPs are added during the update.
1492                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1493                     addedExternalIps.removeAll(originalExternalIps);
1494                     if (addedExternalIps.size() != 0) {
1495                         LOG.debug("update : Start processing of the External IPs addition for router {}",
1496                             routerName);
1497                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1498                                 update.getExtGwMacAddress(), dpnId,
1499                                 update.getNetworkId());
1500
1501                         for (String addedExternalIp : addedExternalIps) {
1502                 /*
1503                     1) Do nothing in the IntExtIp model.
1504                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1505                  */
1506                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1507                             String externalIp = externalIpParts[0];
1508                             String externalIpPrefix = externalIpParts[1];
1509                             String externalpStr = externalIp + "/" + externalIpPrefix;
1510                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1511                                             + "router ID {} in the ExternalIpsCounter model.",
1512                                     externalpStr, routerId);
1513                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1514                             subnetRegisterMapping(update, routerId);
1515                             LOG.info("update : Installing fib flow fo newly added Ips");
1516                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1517                         }
1518                         LOG.debug(
1519                                 "update : End processing of the External IPs addition during the update operation");
1520                     }
1521
1522                     //Check if its Update on subnets
1523                     LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1524                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1525                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1526                     Set<Uuid> addedSubnetIds =
1527                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1528                     if (originalSubnetIds != null) {
1529                         addedSubnetIds.removeAll(originalSubnetIds);
1530                     }
1531
1532                     //Check if the Subnet IDs are added during the update.
1533                     if (addedSubnetIds.size() != 0) {
1534                         LOG.debug(
1535                                 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1536                         for (Uuid addedSubnetId : addedSubnetIds) {
1537                 /*
1538                     1) Select the least loaded external IP for the subnet and store the mapping of the
1539                     subnet IP and the external IP in the IntExtIp model.
1540                     2) Increase the count of the selected external IP by one.
1541                     3) Advertise to the BGP if external IP is allocated for the first time for the
1542                     router i.e. the route for the external IP is absent.
1543                  */
1544                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1545                             if (subnetIp != null) {
1546                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1547                                         writeFlowInvTx);
1548                             }
1549                         }
1550                         LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1551                     }
1552
1553                     //Check if the Subnet IDs are removed during the update.
1554                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1555                     removedSubnetIds.removeAll(updatedSubnetIds);
1556                     if (removedSubnetIds.size() != 0) {
1557                         LOG.debug(
1558                                 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1559                         for (Uuid removedSubnetId : removedSubnetIds) {
1560                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1561                             if (subnetAddr != null) {
1562                     /*
1563                         1) Remove the subnet IP and the external IP in the IntExtIp map
1564                         2) Decrease the count of the coresponding external IP by one.
1565                         3) Advertise to the BGP for removing the routes of the corresponding external
1566                         IP if its not allocated to any other internal IP.
1567                     */
1568
1569                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1570                                         subnetAddr[0] + "/" + subnetAddr[1]);
1571                                 if (externalIp == null) {
1572                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1573                                             routerId, subnetAddr[0]);
1574                                     return;
1575                                 }
1576
1577                                 naptManager.updateCounter(routerId, externalIp, false);
1578                                 // Traverse entire model of external-ip counter whether external ip is not
1579                                 // used by any other internal ip in any router
1580                                 if (!isExternalIpAllocated(externalIp)) {
1581                                     LOG.debug("update : external ip is not allocated to any other "
1582                                             + "internal IP so proceeding to remove routes");
1583                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1584                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1585                                             removeFlowInvTx);
1586                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1587                                                     + "router {} with networkId {} and externalIp {}",
1588                                             dpnId, routerId, networkId, externalIp);
1589                                 }
1590
1591                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1592                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1593                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1594                             }
1595                         }
1596                         LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1597                     }
1598                 }));
1599             }));
1600             return futures;
1601         }, NatConstants.NAT_DJC_MAX_RETRIES);
1602     }
1603
1604     private boolean isExternalIpAllocated(String externalIp) {
1605         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1606         Optional<ExternalIpsCounter> externalCountersData;
1607         try {
1608             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1609                         LogicalDatastoreType.OPERATIONAL, id);
1610         } catch (ReadFailedException e) {
1611             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1612             externalCountersData = Optional.absent();
1613         }
1614         if (externalCountersData.isPresent()) {
1615             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1616             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1617                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1618                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1619                         if (externalIpCount.getCounter() != 0) {
1620                             return true;
1621                         }
1622                         break;
1623                     }
1624                 }
1625             }
1626         }
1627         return false;
1628     }
1629
1630     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1631             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1632         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1633         try {
1634             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1635             if (address instanceof Inet6Address) {
1636                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1637                 return;
1638             }
1639         } catch (UnknownHostException e) {
1640             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1641             return;
1642         }
1643         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1644         if (leastLoadedExtIpAddr != null) {
1645             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1646             String leastLoadedExtIp = externalIpParts[0];
1647             String leastLoadedExtIpPrefix = externalIpParts[1];
1648             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1649             subnetIp = subnetIpParts[0];
1650             String subnetIpPrefix = subnetIpParts[1];
1651             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1652             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1653                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1654                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1655             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1656
1657
1658             // Check if external IP is already assigned a route. (i.e. External IP is previously
1659             // allocated to any of the subnets)
1660             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1661             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1662             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1663             if (label != null) {
1664                 //update
1665                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1666                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1667                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1668                     label, internalIp, leastLoadedExtIpAddrStr);
1669                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1670                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1671                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1672                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1673                 return;
1674             }
1675
1676             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1677             // for the first time and hence not having a route.
1678             //Get the VPN Name using the network ID
1679             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1680             if (vpnName != null) {
1681                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1682                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1683                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1684                         + "added after gateway-set");
1685                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1686                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1687                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1688                         return;
1689                     }
1690                 }
1691                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1692                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1693                     writeFlowInvTx);
1694             }
1695         }
1696     }
1697
1698     @Nullable
1699     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1700         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1701         for (IpMap ipMap : ipMaps) {
1702             if (ipMap.getExternalIp().equals(externalIp)) {
1703                 if (ipMap.getLabel() != null) {
1704                     return ipMap.getLabel();
1705                 }
1706             }
1707         }
1708         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1709         return null;
1710     }
1711
1712     @Override
1713     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1714         LOG.trace("remove : Router delete method");
1715         /*
1716         ROUTER DELETE SCENARIO
1717         1) Get the router ID from the event.
1718         2) Build the cookie information from the router ID.
1719         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1720         4) Build the flow with the cookie value.
1721         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1722         6) Remove the flows from the other switches which points to the primary and secondary
1723          switches for the flows related the router ID.
1724         7) Get the list of external IP address maintained for the router ID.
1725         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1726         9) Withdraw the corresponding routes from the BGP.
1727          */
1728
1729         if (identifier == null || router == null) {
1730             LOG.error("remove : returning without processing since routers is null");
1731             return;
1732         }
1733
1734         String routerName = router.getRouterName();
1735         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1736             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1737                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1738                         routerName);
1739                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1740                 if (routerId == NatConstants.INVALID_ID) {
1741                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1742                             routerName);
1743                     return;
1744                 }
1745                 long bgpVpnId = NatConstants.INVALID_ID;
1746                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1747                 if (bgpVpnUuid != null) {
1748                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1749                 }
1750                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1751                         tx);
1752                 Uuid networkUuid = router.getNetworkId();
1753
1754                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1755                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1756                     // No NAPT switch for external router, probably because the router is not attached to
1757                     // any
1758                     // internal networks
1759                     LOG.debug(
1760                             "No NAPT switch for router {}, check if router is attached to any internal "
1761                                     + "network",
1762                             routerName);
1763                     return;
1764                 } else {
1765                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1766                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1767                             routerId, tx);
1768                 }
1769                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1770                     == NatConstants.INVALID_ID) {
1771                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1772                 }
1773             })), NatConstants.NAT_DJC_MAX_RETRIES);
1774     }
1775
1776     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1777                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1778                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1779         LOG.info("handleDisableSnat : Entry");
1780         String routerName = router.getRouterName();
1781         try {
1782             if (routerFlag) {
1783                 removeNaptSwitch(routerName);
1784             } else {
1785                 updateNaptSwitch(routerName, BigInteger.ZERO);
1786             }
1787
1788             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1789             naptManager.removeExternalCounter(routerId);
1790
1791             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1792             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1793                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1794                     + "router ID {} from RouterNaptSwitch model", routerId);
1795                 return;
1796             }
1797             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1798                     networkUuid);
1799             if (extNwProvType == null) {
1800                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1801                 return;
1802             }
1803             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1804             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1805                     externalSubnetList, removeFlowInvTx, extNwProvType);
1806             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1807             String externalSubnetVpn = null;
1808             for (Uuid externalSubnetId : externalSubnetList) {
1809                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1810                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1811                 if (externalSubnet.isPresent()) {
1812                     externalSubnetVpn =  externalSubnetId.getValue();
1813                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1814                             router.getExtGwMacAddress(), removeFlowInvTx);
1815                 }
1816             }
1817             if (externalSubnetVpn == null) {
1818                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1819                         router.getExtGwMacAddress(), removeFlowInvTx);
1820             }
1821             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1822             // for the router ID.
1823             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1824                     + "router ID {} in the DS", routerId);
1825             naptManager.removeMapping(routerId);
1826         } catch (InterruptedException | ExecutionException e) {
1827             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1828         }
1829         LOG.info("handleDisableSnat : Exit");
1830     }
1831
1832     // TODO Clean up the exception handling
1833     @SuppressWarnings("checkstyle:IllegalCatch")
1834     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1835                                              @NonNull Collection<String> externalIps,
1836                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1837         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1838                 + "with internet vpn {}", routerName, vpnId);
1839         try {
1840             BigInteger naptSwitchDpnId = null;
1841             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1842                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1843             Optional<RouterToNaptSwitch> rtrToNapt;
1844             try {
1845                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1846                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1847             } catch (ReadFailedException e) {
1848                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1849                 rtrToNapt = Optional.absent();
1850             }
1851             if (rtrToNapt.isPresent()) {
1852                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1853             }
1854             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1855
1856             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1857                     writeFlowInvTx);
1858             try {
1859                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1860                 if (extGwMacAddress != null) {
1861                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1862                             + "External Router ID {}", extGwMacAddress, routerId);
1863                 } else {
1864                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1865                             + "External Router ID {}", routerId);
1866                     return;
1867                 }
1868                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1869                         writeFlowInvTx);
1870             } catch (Exception ex) {
1871                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1872                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1873             }
1874             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1875                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1876             }
1877         } catch (InterruptedException | ExecutionException e) {
1878             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1879                     + "with internet vpn {}", routerName, vpnId, e);
1880         }
1881         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1882                 routerName, vpnId);
1883     }
1884
1885     // TODO Clean up the exception handling
1886     @SuppressWarnings("checkstyle:IllegalCatch")
1887     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1888         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1889             .setPrimarySwitchId(naptSwitchId).build();
1890         try {
1891             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1892                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1893         } catch (Exception ex) {
1894             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1895                 naptSwitchId, routerName);
1896         }
1897         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1898             naptSwitchId, routerName);
1899     }
1900
1901     protected void removeNaptSwitch(String routerName) {
1902         // Remove router and switch from model
1903         InstanceIdentifier<RouterToNaptSwitch> id =
1904             InstanceIdentifier.builder(NaptSwitches.class)
1905                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1906         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1907         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1908         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1909         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1910     }
1911
1912     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1913                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1914                                                 @NonNull Collection<String> externalIps,
1915                                                 Collection<Uuid> externalSubnetList,
1916                                                 TypedReadWriteTransaction<Configuration> confTx,
1917                                                 ProviderTypes extNwProvType)
1918             throws InterruptedException, ExecutionException {
1919
1920         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1921         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1922
1923         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1924         // traffic which comes from the  VMs of the NAPT switches)
1925         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1926         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1927
1928         LOG.info(
1929             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1930                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1931         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1932
1933         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1934         // traffic which comes from the VMs of the non NAPT switches)
1935         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1936                 elanManager, idManager, routerId, routerName);
1937         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1938         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1939         LOG.info(
1940             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1941                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1942         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1943
1944         //Remove the flow table 25->44 from NAPT Switch
1945         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1946             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1947         }
1948
1949         //Remove the Outbound flow entry which forwards the packet to FIB Table
1950         LOG.info(
1951             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1952                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1953
1954         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1955             NwConstants.IP_PROT_TCP);
1956         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1957             outboundTcpNatFlowRef);
1958         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1959
1960         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1961             NwConstants.IP_PROT_UDP);
1962         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1963             outboundUdpNatFlowRef);
1964         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1965
1966         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1967             NwConstants.IP_PROT_ICMP);
1968         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1969             icmpDropFlowRef);
1970         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1971         boolean lastRouterOnExternalNetwork =
1972             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1973         if (lastRouterOnExternalNetwork) {
1974             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1975         }
1976         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1977         // External Subnet Vpn Id.
1978         for (Uuid externalSubnetId : externalSubnetList) {
1979             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1980             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1981                 dataBroker, externalSubnetId, routerName, dpnId)) {
1982                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1983                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1984                     natPfibSubnetFlowRef);
1985                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1986                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1987                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1988                     subnetVpnId, dpnId);
1989             }
1990         }
1991
1992         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1993         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1994         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1995
1996         LOG.info(
1997             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1998             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1999         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2000
2001         if (lastRouterOnExternalNetwork) {
2002             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2003             // - This does not work since ext-routers is deleted already - no network info
2004             //Get the VPN ID from the ExternalNetworks model
2005             long vpnId = -1;
2006             if (vpnName == null || vpnName.isEmpty()) {
2007                 // ie called from router delete cases
2008                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2009                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2010                 if (vpnUuid != null) {
2011                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2012                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2013                               + "or disableSNAT scenario", vpnId, networkId);
2014                 }
2015             } else {
2016                 // ie called from disassociate vpn case
2017                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2018                     vpnName);
2019                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2020                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2021                     vpnId);
2022             }
2023
2024             if (vpnId != NatConstants.INVALID_ID) {
2025                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2026                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2027                 FlowEntity natPfibVpnFlowEntity =
2028                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2029                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2030                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2031                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2032             }
2033         }
2034
2035         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2036         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2037         if (ipPortMapping == null) {
2038             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2039             return;
2040         }
2041
2042         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2043             String protocol = intextIpProtocolType.getProtocol().name();
2044             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2045                 String ipPortInternal = ipPortMap.getIpPortInternal();
2046                 String[] ipPortParts = ipPortInternal.split(":");
2047                 if (ipPortParts.length != 2) {
2048                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2049                     return;
2050                 }
2051                 String internalIp = ipPortParts[0];
2052                 String internalPort = ipPortParts[1];
2053
2054                 //Build the flow for the outbound NAPT table
2055                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2056                     + NatConstants.COLON_SEPARATOR + internalPort);
2057                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2058                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2059                 FlowEntity outboundNaptFlowEntity =
2060                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2061
2062                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2063                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2064                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2065
2066                  //Build the flow for the inbound NAPT table
2067                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2068                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2069                 FlowEntity inboundNaptFlowEntity =
2070                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2071
2072                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2073                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2074                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2075             }
2076         }
2077     }
2078
2079     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2080                                                     @NonNull Collection<String> externalIps,
2081                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2082             throws ExecutionException, InterruptedException {
2083         long extVpnId = NatConstants.INVALID_ID;
2084         if (networkId != null) {
2085             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2086             if (vpnUuid != null) {
2087                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2088             } else {
2089                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2090             }
2091         } else {
2092             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2093             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2094         }
2095         if (extVpnId == NatConstants.INVALID_ID) {
2096             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2097             extVpnId = routerId;
2098         }
2099         for (String ip : externalIps) {
2100             String extIp = removeMaskFromIp(ip);
2101             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2102             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2103                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2104                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2105             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2106             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2107         }
2108     }
2109
2110     private String removeMaskFromIp(String ip) {
2111         if (ip != null && !ip.trim().isEmpty()) {
2112             return ip.split("/")[0];
2113         }
2114         return ip;
2115     }
2116
2117     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2118                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2119                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2120             throws ExecutionException, InterruptedException {
2121         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2122         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2123
2124         //Remove the NAPT PFIB TABLE entry
2125         long vpnId = -1;
2126         if (vpnName != null) {
2127             // ie called from disassociate vpn case
2128             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2129                     + "with vpnName {}", vpnName);
2130             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2131             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2132                     vpnId);
2133         }
2134
2135         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2136                 networkId, routerName, dpnId)) {
2137             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2138             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2139             FlowEntity natPfibVpnFlowEntity =
2140                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2141             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2142                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2143             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2144
2145             // Remove IP-PORT active NAPT entries and release port from IdManager
2146             // For the router ID get the internal IP , internal port and the corresponding
2147             // external IP and external Port.
2148             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2149             if (ipPortMapping == null) {
2150                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2151                 return;
2152             }
2153             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2154                 String protocol = intextIpProtocolType.getProtocol().name();
2155                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2156                     String ipPortInternal = ipPortMap.getIpPortInternal();
2157                     String[] ipPortParts = ipPortInternal.split(":");
2158                     if (ipPortParts.length != 2) {
2159                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2160                                 + "and port");
2161                         return;
2162                     }
2163                     String internalIp = ipPortParts[0];
2164                     String internalPort = ipPortParts[1];
2165
2166                     //Build the flow for the outbound NAPT table
2167                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2168                             + NatConstants.COLON_SEPARATOR + internalPort);
2169                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2170                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2171                     FlowEntity outboundNaptFlowEntity =
2172                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2173
2174                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2175                             + "active switch with the DPN ID {} and router ID {}",
2176                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2177                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2178
2179                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2180                     final String externalIp = ipPortExternal.getIpAddress();
2181
2182                     //Build the flow for the inbound NAPT table
2183                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2184                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2185                     FlowEntity inboundNaptFlowEntity =
2186                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2187
2188                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2189                             + "active active switch with the DPN ID {} and router ID {}",
2190                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2191                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2192
2193                     // Finally release port from idmanager
2194                     String internalIpPort = internalIp + ":" + internalPort;
2195                     naptManager.removePortFromPool(internalIpPort, externalIp);
2196
2197                     //Remove sessions from models
2198                     naptManager.removeIpPortMappingForRouterID(routerId);
2199                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2200                 }
2201             }
2202         } else {
2203             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2204         }
2205     }
2206
2207     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2208             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2209             throws ExecutionException, InterruptedException {
2210         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2211
2212         // Remove the flows from the other switches which points to the primary and secondary switches
2213         // for the flows related the router ID.
2214         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2215         if (allSwitchList.isEmpty()) {
2216             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2217             return;
2218         }
2219         for (BigInteger dpnId : allSwitchList) {
2220             if (!naptSwitchDpnId.equals(dpnId)) {
2221                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2222
2223                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2224                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2225                 FlowEntity preSnatFlowEntity =
2226                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2227
2228                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2229                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2230                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2231
2232                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2233                 long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2234                     NatUtil.getGroupIdKey(routerName));
2235                 if (groupId != NatConstants.INVALID_ID) {
2236                     LOG.info(
2237                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2238                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2239                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2240                 } else {
2241                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2242                 }
2243             }
2244         }
2245     }
2246
2247     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2248                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2249                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2250             throws ExecutionException, InterruptedException {
2251         //Withdraw the corresponding routes from the BGP.
2252         //Get the network ID using the router ID.
2253         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2254                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2255         if (networkUuid == null) {
2256             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2257             return;
2258         }
2259
2260         if (externalIps.isEmpty()) {
2261             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2262             return;
2263         }
2264
2265         if (vpnName == null) {
2266             //Get the VPN Name using the network ID
2267             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2268             if (vpnName == null) {
2269                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2270                     networkUuid, routerId);
2271                 return;
2272             }
2273         }
2274         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2275
2276         //Remove custom FIB routes
2277         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2278         for (String extIp : externalIps) {
2279             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2280         }
2281     }
2282
2283     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2284                                             final Uuid networkUuid, String extGwMacAddress,
2285                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2286             throws ExecutionException, InterruptedException {
2287         clearBgpRoutes(extIp, vpnName);
2288         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2289                 removeFlowInvTx);
2290     }
2291
2292     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2293                                              String vpnName, Uuid extNetworkId, long tempLabel,
2294                                              String gwMacAddress, boolean switchOver,
2295                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2296             throws ExecutionException, InterruptedException {
2297         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2298         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2299         if (routerName == null) {
2300             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2301             return;
2302         }
2303         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2304         if (extNwProvType == null) {
2305             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2306             return;
2307         }
2308         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2309          * external network provided type is VxLAN
2310          */
2311         if (extNwProvType == ProviderTypes.VXLAN) {
2312             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2313             );
2314             return;
2315         }
2316         if (tempLabel < 0) {
2317             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2318                     extIp, routerId);
2319             return;
2320         }
2321         final long label = tempLabel;
2322         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2323         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2324             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2325             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2326             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2327                     extRouter);
2328
2329             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2330                     externalSubnetId);
2331
2332             if (externalSubnet.isPresent()) {
2333                 vpnName =  externalSubnetId.getValue();
2334             }
2335         }
2336         final String externalVpn = vpnName;
2337         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2338                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2339                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2340         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2341
2342         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2343         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2344         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2345             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2346             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2347         }
2348         if (!switchOver) {
2349             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2350                 Futures.transformAsync(future, result -> {
2351                     //Release label
2352                     if (result.isSuccessful()) {
2353                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2354                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2355                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2356                         return vpnService.removeVpnLabel(labelInput);
2357                     } else {
2358                         String errMsg =
2359                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2360                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2361                         LOG.error(errMsg);
2362                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2363                     }
2364                 }, MoreExecutors.directExecutor());
2365
2366             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2367
2368                 @Override
2369                 public void onFailure(@NonNull Throwable error) {
2370                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2371                         + "got external ip {}", label, extIp, error);
2372                 }
2373
2374                 @Override
2375                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2376                     if (result.isSuccessful()) {
2377                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2378                             + "from VPN {}", externalIp, externalVpn);
2379                     } else {
2380                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2381                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2382                     }
2383                 }
2384             }, MoreExecutors.directExecutor());
2385         } else {
2386             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2387                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2388         }
2389     }
2390
2391     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2392                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2393                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2394             throws ExecutionException, InterruptedException {
2395         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2396         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2397         if (routerName == null) {
2398             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2399             return;
2400         }
2401         //Get the external network provider type from networkId
2402         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2403         if (extNwProvType == null) {
2404             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2405                     networkUuid);
2406             return;
2407         }
2408         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2409          *  external network provided type is VxLAN
2410          */
2411         if (extNwProvType == ProviderTypes.VXLAN) {
2412             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2413             return;
2414         }
2415         //Get IPMaps from the DB for the router ID
2416         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2417         if (dbIpMaps.isEmpty()) {
2418             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2419             return;
2420         }
2421
2422         long tempLabel = NatConstants.INVALID_ID;
2423         for (IpMap dbIpMap : dbIpMaps) {
2424             String dbExternalIp = dbIpMap.getExternalIp();
2425             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2426             //Select the IPMap, whose external IP is the IP for which FIB is installed
2427             if (extIp.equals(dbExternalIp)) {
2428                 tempLabel = dbIpMap.getLabel();
2429                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2430                     tempLabel, dbExternalIp, routerId);
2431                 break;
2432             }
2433         }
2434         if (tempLabel == NatConstants.INVALID_ID) {
2435             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2436                     extIp, routerId);
2437             return;
2438         }
2439
2440         final long label = tempLabel;
2441         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2442         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2443                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2444                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2445         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2446
2447         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2448         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2449         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2450             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2451             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2452         }
2453         if (!switchOver) {
2454             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2455                     Futures.transformAsync(future, result -> {
2456                         //Release label
2457                         if (result.isSuccessful()) {
2458                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2459                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2460                             return vpnService.removeVpnLabel(labelInput);
2461                         } else {
2462                             String errMsg =
2463                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2464                                             + "prefix %s Failed - %s",
2465                                             dpnId, externalIp, result.getErrors());
2466                             LOG.error(errMsg);
2467                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2468                         }
2469                     }, MoreExecutors.directExecutor());
2470
2471             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2472
2473                 @Override
2474                 public void onFailure(@NonNull Throwable error) {
2475                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2476                 }
2477
2478                 @Override
2479                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2480                     if (result.isSuccessful()) {
2481                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2482                             + "from VPN {}", externalIp, vpnName);
2483                     } else {
2484                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2485                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2486                     }
2487                 }
2488             }, MoreExecutors.directExecutor());
2489         } else {
2490             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2491                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2492         }
2493     }
2494
2495     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2496             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2497             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2498         //Withdraw the corresponding routes from the BGP.
2499         //Get the network ID using the router ID.
2500         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2501                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2502         if (networkUuid == null) {
2503             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2504             return;
2505         }
2506
2507         if (externalIps == null || externalIps.isEmpty()) {
2508             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2509             return;
2510         }
2511
2512         if (vpnName == null) {
2513             //Get the VPN Name using the network ID
2514             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2515             if (vpnName == null) {
2516                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2517                     networkUuid, routerId);
2518                 return;
2519             }
2520         }
2521         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2522
2523         //Remove custom FIB routes
2524         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2525         for (String extIp : externalIps) {
2526             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2527                     writeFlowInvTx);
2528         }
2529     }
2530
2531     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2532         //Inform BGP about the route removal
2533         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2534         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2535         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2536     }
2537
2538     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2539             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2540         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2541         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2542             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2543         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2544     }
2545
2546     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2547             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2548         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2549         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2550         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2551         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2552     }
2553
2554     /**
2555      * router association to vpn.
2556      *
2557      * @param routerName - Name of router
2558      * @param routerId - router id
2559      * @param bgpVpnName BGP VPN name
2560      */
2561     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2562             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2563         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2564         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2565             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2566
2567             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2568
2569             if (bgpVpnId != NatConstants.INVALID_ID) {
2570                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2571                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2572                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2573                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2574                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2575                     getRoutersIdentifier(bgpVpnId), rtrs);
2576
2577                 // Get the allocated Primary NAPT Switch for this router
2578                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2579                         routerId, bgpVpnId);
2580                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2581
2582                 // Get the group ID
2583                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2584                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2585                         extNwProvType);
2586             }
2587         }
2588     }
2589
2590     /**
2591      * router disassociation from vpn.
2592      *
2593      * @param routerName - Name of router
2594      * @param routerId - router id
2595      * @param bgpVpnName BGP VPN name
2596      */
2597     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2598             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2599         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2600         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2601             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2602             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2603
2604             // Get the allocated Primary NAPT Switch for this router
2605             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2606
2607             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2608             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2609
2610             // Get the group ID
2611             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2612             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2613                     writeFlowInvTx, extNwProvType);
2614         }
2615     }
2616
2617     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2618         InstanceIdentifier<Routers> routerInstanceIndentifier =
2619             InstanceIdentifier.builder(ExtRouters.class)
2620                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2621         try {
2622             Optional<Routers> routerData = SingleTransactionDataBroker
2623                     .syncReadOptional(dataBroker,
2624                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2625             return routerData.isPresent() && routerData.get().isEnableSnat();
2626         } catch (ReadFailedException e) {
2627             LOG.error("Failed to read data for router id {}", routerUuid, e);
2628             return false;
2629         }
2630     }
2631
2632     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2633         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2634
2635         long changedVpnId = bgpVpnId;
2636         String idType = "BGP VPN";
2637         if (bgpVpnId == NatConstants.INVALID_ID) {
2638             changedVpnId = routerId;
2639             idType = "router";
2640         }
2641
2642         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2643         if (switches.isEmpty()) {
2644             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2645             return;
2646         }
2647         for (BigInteger dpnId : switches) {
2648             // Update the BGP VPN ID in the SNAT miss entry to group
2649             if (!dpnId.equals(primarySwitchId)) {
2650                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2651                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2652                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2653                 long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2654                     NatUtil.getGroupIdKey(routerName));
2655                 if (groupId != NatConstants.INVALID_ID) {
2656                     if (!isSnatCfgd) {
2657                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2658                     }
2659                     LOG.debug(
2660                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2661                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2662                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2663                         groupId, changedVpnId);
2664                     mdsalManager.addFlow(confTx, flowEntity);
2665                 } else {
2666                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2667                 }
2668             } else {
2669                 LOG.debug(
2670                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2671                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2672                 FlowEntity flowEntity =
2673                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2674                 mdsalManager.addFlow(confTx, flowEntity);
2675
2676                 LOG.debug(
2677                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2678                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2679                         idType, changedVpnId, primarySwitchId);
2680                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2681                         changedVpnId, confTx, extNwProvType);
2682
2683                 LOG.debug(
2684                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2685                                 + "which punts the packet to the controller in the Primary switch {}",
2686                         idType, changedVpnId, primarySwitchId);
2687                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2688
2689                 LOG.debug(
2690                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2691                                 + " outgoing packet to FIB Table in the Primary switch {}",
2692                         idType, changedVpnId, primarySwitchId);
2693                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2694
2695                 LOG.debug(
2696                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2697                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2698                                 + " {}", idType, changedVpnId, primarySwitchId);
2699                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2700
2701                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2702                         primarySwitchId);
2703                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2704                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2705                 if (vpnId != NatConstants.INVALID_ID) {
2706                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2707                 }
2708             }
2709         }
2710     }
2711
2712     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2713         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2714         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2715         if (ipPortMapping == null) {
2716             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2717             return;
2718         }
2719         // Get the External Gateway MAC Address
2720         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2721         if (extGwMacAddress != null) {
2722             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2723                     extGwMacAddress, routerId);
2724         } else {
2725             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2726                     routerId);
2727             return;
2728         }
2729         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2730             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2731                 String ipPortInternal = ipPortMap.getIpPortInternal();
2732                 String[] ipPortParts = ipPortInternal.split(":");
2733                 if (ipPortParts.length != 2) {
2734                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2735                     return;
2736                 }
2737                 String internalIp = ipPortParts[0];
2738                 String internalPort = ipPortParts[1];
2739                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2740                         internalIp, internalPort);
2741                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2742                 NAPTEntryEvent.Protocol protocol;
2743                 switch (protocolTypes) {
2744                     case TCP:
2745                         protocol = NAPTEntryEvent.Protocol.TCP;
2746                         break;
2747                     case UDP:
2748                         protocol = NAPTEntryEvent.Protocol.UDP;
2749                         break;
2750                     default:
2751                         protocol = NAPTEntryEvent.Protocol.TCP;
2752                 }
2753                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2754                 SessionAddress externalAddress =
2755                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2756                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2757                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2758                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2759                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2760                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2761             }
2762         }
2763     }
2764
2765     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2766                                                           long changedVpnId) {
2767
2768         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2769             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2770         List<MatchInfo> matches = new ArrayList<>();
2771         matches.add(MatchEthernetType.IPV4);
2772         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2773
2774         List<ActionInfo> actionsInfo = new ArrayList<>();
2775         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2776                 elanManager, idManager, changedVpnId, routerName);
2777         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2778         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2779                 actionsInfo);
2780         actionsInfo.add(new ActionGroup(groupId));
2781         List<InstructionInfo> instructions = new ArrayList<>();
2782         instructions.add(new InstructionApplyActions(actionsInfo));
2783         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2784         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2785             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2786             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2787
2788         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2789         return flowEntity;
2790     }
2791
2792     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2793                                                                         long changedVpnId) {
2794
2795         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2796                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2797         List<MatchInfo> matches = new ArrayList<>();
2798         matches.add(MatchEthernetType.IPV4);
2799         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2800
2801         List<InstructionInfo> instructions = new ArrayList<>();
2802         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2803
2804         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2805         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2806             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2807             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2808
2809         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2810         return flowEntity;
2811     }
2812
2813     // TODO : Replace this with ITM Rpc once its available with full functionality
2814     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2815         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2816
2817         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2818             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2819         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2820                 extNwProvType);
2821         mdsalManager.addFlow(confTx, flowEntity);
2822     }
2823
2824     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2825             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2826         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2827             dpId, routerName, changedVpnId);
2828         List<MatchInfo> matches = new ArrayList<>();
2829         matches.add(MatchEthernetType.IPV4);
2830
2831         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2832         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2833             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2834         }
2835         matches.add(new MatchTunnelId(tunnelId));
2836
2837         List<InstructionInfo> instructions = new ArrayList<>();
2838         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2839             MetaDataUtil.METADATA_MASK_VRFID));
2840         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2841         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2842         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2843         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2844             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2845             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2846         return flowEntity;
2847     }
2848
2849     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2850                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2851         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2852             dpnId, routerId, changedVpnId);
2853         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2854                 NwConstants.IP_PROT_TCP);
2855         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2856         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2857
2858         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2859                 NwConstants.IP_PROT_UDP);
2860         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2861         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2862
2863         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2864         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2865         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2866     }
2867
2868     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2869                                                            long changedVpnId, int protocol) {
2870         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2871             dpId, routerId, changedVpnId);
2872         BigInteger cookie = getCookieOutboundFlow(routerId);
2873         List<MatchInfo> matches = new ArrayList<>();
2874         matches.add(MatchEthernetType.IPV4);
2875         matches.add(new MatchIpProtocol((short)protocol));
2876         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2877
2878         List<InstructionInfo> instructions = new ArrayList<>();
2879         List<ActionInfo> actionsInfos = new ArrayList<>();
2880         actionsInfos.add(new ActionPuntToController());
2881         if (snatPuntTimeout != 0) {
2882             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2883         }
2884         instructions.add(new InstructionApplyActions(actionsInfos));
2885
2886         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2887         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2888             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2889         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2890         return flowEntity;
2891     }
2892
2893     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2894                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2895         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2896             dpnId, segmentId, changedVpnId);
2897         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2898         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2899     }
2900
2901     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2902
2903         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2904             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2905         List<MatchInfo> matches = new ArrayList<>();
2906         matches.add(MatchEthernetType.IPV4);
2907         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2908
2909         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2910         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2911         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2912         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2913         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2914
2915         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2916         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2917             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2918             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2919         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2920         return flowEntity;
2921     }
2922
2923     @Override
2924     protected ExternalRoutersListener getDataTreeChangeListener() {
2925         return ExternalRoutersListener.this;
2926     }
2927
2928     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2929                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2930         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2931                 routerName);
2932         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2933             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2934             if (subnetVpnId != -1) {
2935                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2936                     + "and vpnId {}", dpnId, subnetVpnId);
2937                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2938             }
2939         }
2940     }
2941 }