natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 145 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 146 import org.opendaylight.yangtools.yang.common.RpcResult;
 147 import org.slf4j.Logger;
 148 import org.slf4j.LoggerFactory;
 149
 150
 151 @Singleton
 152 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 153     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 154
 155     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 156     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 157
 158     private final DataBroker dataBroker;
 159     private final ManagedNewTransactionRunner txRunner;
 160     private final IMdsalApiManager mdsalManager;
 161     private final ItmRpcService itmManager;
 162     private final OdlInterfaceRpcService odlInterfaceRpcService;
 163     private final IdManagerService idManager;
 164     private final NaptManager naptManager;
 165     private final NAPTSwitchSelector naptSwitchSelector;
 166     private final IBgpManager bgpManager;
 167     private final VpnRpcService vpnService;
 168     private final FibRpcService fibService;
 169     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 170     private final NaptEventHandler naptEventHandler;
 171     private final NaptPacketInHandler naptPacketInHandler;
 172     private final IFibManager fibManager;
 173     private final IVpnManager vpnManager;
 174     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 175     private final NatMode natMode;
 176     private final IElanService elanManager;
 177     private final JobCoordinator coordinator;
 178     private final IInterfaceManager interfaceManager;
 179     private final NatOverVxlanUtil natOverVxlanUtil;
 180     private final int snatPuntTimeout;
 181
 182     @Inject
 183     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 184                                    final ItmRpcService itmManager,
 185                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 186                                    final IdManagerService idManager,
 187                                    final NaptManager naptManager,
 188                                    final NAPTSwitchSelector naptSwitchSelector,
 189                                    final IBgpManager bgpManager,
 190                                    final VpnRpcService vpnService,
 191                                    final FibRpcService fibService,
 192                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 193                                    final NaptEventHandler naptEventHandler,
 194                                    final NaptPacketInHandler naptPacketInHandler,
 195                                    final IFibManager fibManager,
 196                                    final IVpnManager vpnManager,
 197                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 198                                    final NatserviceConfig config,
 199                                    final IElanService elanManager,
 200                                    final JobCoordinator coordinator,
 201                                    final NatOverVxlanUtil natOverVxlanUtil,
 202                                    final IInterfaceManager interfaceManager) {
 203         super(Routers.class, ExternalRoutersListener.class);
 204         this.dataBroker = dataBroker;
 205         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 206         this.mdsalManager = mdsalManager;
 207         this.itmManager = itmManager;
 208         this.odlInterfaceRpcService = odlInterfaceRpcService;
 209         this.idManager = idManager;
 210         this.naptManager = naptManager;
 211         this.naptSwitchSelector = naptSwitchSelector;
 212         this.bgpManager = bgpManager;
 213         this.vpnService = vpnService;
 214         this.fibService = fibService;
 215         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 216         this.naptEventHandler = naptEventHandler;
 217         this.naptPacketInHandler = naptPacketInHandler;
 218         this.fibManager = fibManager;
 219         this.vpnManager = vpnManager;
 220         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 221         this.elanManager = elanManager;
 222         this.coordinator = coordinator;
 223         this.interfaceManager = interfaceManager;
 224         this.natOverVxlanUtil = natOverVxlanUtil;
 225         if (config != null) {
 226             this.natMode = config.getNatMode();
 227             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 228         } else {
 229             this.natMode = NatMode.Controller;
 230             this.snatPuntTimeout = 0;
 231         }
 232     }
 233
 234     @Override
 235     @PostConstruct
 236     public void init() {
 237         LOG.info("{} init", getClass().getSimpleName());
 238         // This class handles ExternalRouters for Controller SNAT mode.
 239         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 240         if (natMode == NatMode.Controller) {
 241             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 242             NatUtil.createGroupIdPool(idManager);
 243         }
 244     }
 245
 246     @Override
 247     protected InstanceIdentifier<Routers> getWildCardPath() {
 248         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 249     }
 250
 251     @Override
 252     // TODO Clean up the exception handling
 253     @SuppressWarnings("checkstyle:IllegalCatch")
 254     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 255         // Populate the router-id-name container
 256         String routerName = routers.getRouterName();
 257         LOG.info("add : external router event for {}", routerName);
 258         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 259         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 260         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 261         try {
 262             if (routers.isEnableSnat()) {
 263                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 264                     () -> Collections.singletonList(
 265                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 266                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 267                             long bgpVpnId = NatConstants.INVALID_ID;
 268                             if (bgpVpnUuid != null) {
 269                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 270                             }
 271                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 272                             // Allocate Primary Napt Switch for this router
 273                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 274                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 275                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 276                             }
 277                         }
 278                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 279             } else {
 280                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 281             }
 282         } catch (Exception ex) {
 283             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 284                     routerName, ex);
 285         }
 286     }
 287
 288     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 289                                  TypedWriteTransaction<Configuration> confTx) {
 290         String routerName = routers.getRouterName();
 291         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 292
 293         naptManager.initialiseExternalCounter(routers, routerId);
 294         subnetRegisterMapping(routers, routerId);
 295
 296         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 297             primarySwitchId, routerName);
 298
 299         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 300                 routers.getNetworkId());
 301         if (extNwProvType == null) {
 302             LOG.error("handleEnableSnat : External Network Provider Type missing");
 303             return;
 304         }
 305
 306         if (bgpVpnId != NatConstants.INVALID_ID) {
 307             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 308                     extNwProvType);
 309         } else {
 310             // write metadata and punt
 311             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 312             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, confTx);
 313             // Now install entries in SNAT tables to point to Primary for each router
 314             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 315             for (BigInteger dpnId : switches) {
 316                 // Handle switches and NAPT switches separately
 317                 if (!dpnId.equals(primarySwitchId)) {
 318                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 319                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 320                 }
 321             }
 322         }
 323
 324         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 325         if (externalIps.isEmpty()) {
 326             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 327             return;
 328         } else {
 329             for (String externalIpAddrPrefix : externalIps) {
 330                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 331                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 332                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 333                 );
 334             }
 335         }
 336         LOG.debug("handleEnableSnat : Exit");
 337     }
 338
 339     private BigInteger getPrimaryNaptSwitch(String routerName) {
 340         // Allocate Primary Napt Switch for this router
 341         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 342         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 343             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 344                 primarySwitchId, routerName);
 345             return primarySwitchId;
 346         }
 347         // Allocated an id from VNI pool for the Router.
 348         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 349         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 350         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 351
 352         return primarySwitchId;
 353     }
 354
 355     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 356                                                      TypedWriteTransaction<Configuration> confTx) {
 357         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 358         if (extVpnId == NatConstants.INVALID_ID) {
 359             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 360             extVpnId = routerId;
 361         }
 362         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 363         if (externalIps.isEmpty()) {
 364             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 365                 dpnId, extVpnId);
 366             return;
 367         }
 368         for (String ip : externalIps) {
 369             Uuid subnetId = getSubnetIdForFixedIp(ip);
 370             if (subnetId != null) {
 371                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 372                 if (subnetVpnId != NatConstants.INVALID_ID) {
 373                     extVpnId = subnetVpnId;
 374                 }
 375                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 376                     dpnId, extVpnId, subnetId);
 377                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 378                 if (postNaptFlowEntity != null) {
 379                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 380                 }
 381             }
 382         }
 383     }
 384
 385     @Nullable
 386     private Uuid getSubnetIdForFixedIp(String ip) {
 387         if (ip != null) {
 388             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 389             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 390             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 391         }
 392         LOG.error("getSubnetIdForFixedIp : ip is null");
 393         return null;
 394     }
 395
 396     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 397         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 398         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 399         int counter = 0;
 400         int extIpCounter = externalIps.size();
 401         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 402                 counter, extIpCounter);
 403         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 404         if (subnetIds == null) {
 405             return;
 406         }
 407         for (Uuid subnet : subnetIds) {
 408             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 409             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 410                 .builder(Subnetmaps.class)
 411                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 412                 .build();
 413             Optional<Subnetmap> sn;
 414             try {
 415                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 416                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 417             } catch (ReadFailedException e) {
 418                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 419                 sn = Optional.absent();
 420             }
 421             if (sn.isPresent()) {
 422                 // subnets
 423                 Subnetmap subnetmapEntry = sn.get();
 424                 String subnetString = subnetmapEntry.getSubnetIp();
 425                 String[] subnetSplit = subnetString.split("/");
 426                 String subnetIp = subnetSplit[0];
 427                 try {
 428                     InetAddress address = InetAddress.getByName(subnetIp);
 429                     if (address instanceof Inet6Address) {
 430                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 431                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 432                         continue;
 433                     }
 434                 } catch (UnknownHostException e) {
 435                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 436                     return;
 437                 }
 438                 String subnetPrefix = "0";
 439                 if (subnetSplit.length == 2) {
 440                     subnetPrefix = subnetSplit[1];
 441                 }
 442                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 443                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 444                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 445                 //externalIps
 446                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 447                         counter, extIpCounter);
 448                 if (extIpCounter != 0) {
 449                     if (counter < extIpCounter) {
 450                         String[] ipSplit = externalIps.get(counter).split("/");
 451                         String externalIp = ipSplit[0];
 452                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 453                         if (ipSplit.length == 2) {
 454                             extPrefix = ipSplit[1];
 455                         }
 456                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 457                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 458                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 459                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 460                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 461                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 462                     } else {
 463                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 464                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 465                         String[] ipSplit = externalIps.get(counter).split("/");
 466                         String externalIp = ipSplit[0];
 467                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 468                         if (ipSplit.length == 2) {
 469                             extPrefix = ipSplit[1];
 470                         }
 471                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 472                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 473                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 474                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 475                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 476                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 477                             externalIp, extPrefix);
 478                     }
 479                 }
 480                 counter++;
 481                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 482             } else {
 483                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 484             }
 485         }
 486     }
 487
 488     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 489             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 490             throws ExecutionException, InterruptedException {
 491         //Check if BGP VPN exists. If exists then invoke the new method.
 492         if (bgpVpnId != NatConstants.INVALID_ID) {
 493             if (bgpVpnUuid != null) {
 494                 String bgpVpnName = bgpVpnUuid.getValue();
 495                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 496                     bgpVpnId, bgpVpnName);
 497                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 498                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 499                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 500             }
 501             if (create) {
 502                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 503             } else {
 504                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 505             }
 506             return;
 507         }
 508
 509         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 510         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 511     }
 512
 513     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 514             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 515         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 516         if (switches.isEmpty()) {
 517             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 518             return;
 519         }
 520         if (routerId == NatConstants.INVALID_ID) {
 521             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 522                     + "default NAT route in FIB", routerName);
 523             return;
 524         }
 525         for (BigInteger dpnId : switches) {
 526             if (create) {
 527                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 528                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 529                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 530             } else {
 531                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 532                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 533                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 534             }
 535         }
 536     }
 537
 538     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 539         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 540         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 541         if (dpnIds.isEmpty()) {
 542             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 543                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 544             return;
 545         }
 546         for (BigInteger dpnId : dpnIds) {
 547             if (bgpVpnId != NatConstants.INVALID_ID) {
 548                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 549                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 550                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 551             } else {
 552                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 553                     + "in dpn {} for the internal vpn", routerId, dpnId);
 554                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 555             }
 556         }
 557     }
 558
 559     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 560             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 561             throws ExecutionException, InterruptedException {
 562         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 563         if (dpnIds.isEmpty()) {
 564             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 565                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 566             return;
 567         }
 568         for (BigInteger dpnId : dpnIds) {
 569             if (bgpVpnId != NatConstants.INVALID_ID) {
 570                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 571                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 572                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 573             } else {
 574                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 575                     + "in dpn {} for the internal vpn", routerId, dpnId);
 576                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 577             }
 578         }
 579     }
 580
 581     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 582         TypedWriteTransaction<Configuration> confTx) {
 583         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 584         if (routerId != NatConstants.INVALID_ID) {
 585             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 586                     primarySwitchId, routerId);
 587             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 588         } else {
 589             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 590                 + "createAndInstallMissEntry", routerName);
 591         }
 592     }
 593
 594     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 595         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 596                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 597     }
 598
 599     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 600         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 601                 .FLOWID_SEPARATOR + vpnId;
 602     }
 603
 604     public BigInteger getCookieOutboundFlow(long routerId) {
 605         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 606             BigInteger.valueOf(routerId));
 607     }
 608
 609     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 610         long l4SrcPortField;
 611         long l4DstPortField;
 612         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 613
 614         if (protocol == NwConstants.IP_PROT_TCP) {
 615             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 616             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 617         } else {
 618             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 619             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 620         }
 621         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 622                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 623                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 624                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 625                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 626                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 627                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 628                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 629                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 630                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 631                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 632                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 633                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 634                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 635                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 636
 637         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 638                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 639     }
 640
 641     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 642         List<MatchInfo> matches = new ArrayList<>();
 643         matches.add(MatchEthernetType.IPV4);
 644         matches.add(MatchIpProtocol.ICMP);
 645         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 646
 647         List<ActionInfo> actionInfos = new ArrayList<>();
 648         actionInfos.add(new ActionDrop());
 649
 650         List<InstructionInfo> instructions = new ArrayList<>();
 651         instructions.add(new InstructionApplyActions(actionInfos));
 652
 653         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 654                 NwConstants.IP_PROT_ICMP);
 655         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 656                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 657                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 658         return flow;
 659     }
 660
 661     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 662         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 663         BigInteger cookie = getCookieOutboundFlow(routerId);
 664         List<MatchInfo> matches = new ArrayList<>();
 665         matches.add(MatchEthernetType.IPV4);
 666         matches.add(new MatchIpProtocol((short)protocol));
 667         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 668
 669         List<InstructionInfo> instructions = new ArrayList<>();
 670         List<ActionInfo> actionsInfos = new ArrayList<>();
 671         actionsInfos.add(new ActionPuntToController());
 672         if (snatPuntTimeout != 0) {
 673             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 674         }
 675         instructions.add(new InstructionApplyActions(actionsInfos));
 676
 677         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 678         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 679             5, flowRef, 0, 0,
 680             cookie, matches, instructions);
 681         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 682         return flowEntity;
 683     }
 684
 685     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 686         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 687         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 688         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 689         mdsalManager.addFlow(confTx, tcpflowEntity);
 690
 691         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 692         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 693         mdsalManager.addFlow(confTx, udpflowEntity);
 694
 695         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 696         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 697         mdsalManager.addFlow(confTx, icmpDropFlow);
 698     }
 699
 700     @Nullable
 701     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 702         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 703         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 704         try {
 705             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 706                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 707                     .setSourceDpid(srcDpId)
 708                     .setDestinationDpid(dstDpId)
 709                     .setTunnelType(tunType)
 710                     .build());
 711             rpcResult = result.get();
 712             if (!rpcResult.isSuccessful()) {
 713                 tunType = TunnelTypeGre.class;
 714                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 715                     .setSourceDpid(srcDpId)
 716                     .setDestinationDpid(dstDpId)
 717                     .setTunnelType(tunType)
 718                     .build());
 719                 rpcResult = result.get();
 720                 if (!rpcResult.isSuccessful()) {
 721                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 722                             rpcResult.getErrors());
 723                 } else {
 724                     return rpcResult.getResult().getInterfaceName();
 725                 }
 726                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 727                         rpcResult.getErrors());
 728             } else {
 729                 return rpcResult.getResult().getInterfaceName();
 730             }
 731         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 732             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 733                     + "between {} and {}", srcDpId, dstDpId, e);
 734         }
 735
 736         return null;
 737     }
 738
 739     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 740         TypedWriteTransaction<Configuration> confTx) {
 741
 742         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 743         // Install miss entry pointing to group
 744         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 745         mdsalManager.addFlow(confTx, flowEntity);
 746     }
 747
 748     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 749             String routerName, long routerId) {
 750         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 751             dpnId, bucketInfo.get(0));
 752         // Install the select group
 753         long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 754             NatUtil.getGroupIdKey(routerName));
 755         if (groupId == NatConstants.INVALID_ID) {
 756             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 757             return;
 758         }
 759         GroupEntity groupEntity =
 760             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 761         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 762         mdsalManager.syncInstallGroup(groupEntity);
 763         // Install miss entry pointing to group
 764         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 765         if (flowEntity == null) {
 766             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 767                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 768                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 769             return;
 770         }
 771         mdsalManager.installFlow(flowEntity);
 772     }
 773
 774     void installGroup(BigInteger dpnId, String routerName, long groupId, List<BucketInfo> bucketInfo) {
 775         GroupEntity groupEntity =
 776             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 777         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 778         mdsalManager.syncInstallGroup(groupEntity);
 779     }
 780
 781     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 782         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 783                 dpId, routerName, groupId);
 784         List<MatchInfo> matches = new ArrayList<>();
 785         matches.add(MatchEthernetType.IPV4);
 786         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 787
 788         List<ActionInfo> actionsInfo = new ArrayList<>();
 789         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 790                 routerId, routerName);
 791         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 792         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 793         actionsInfo.add(new ActionGroup(groupId));
 794         List<InstructionInfo> instructions = new ArrayList<>();
 795         instructions.add(new InstructionApplyActions(actionsInfo));
 796         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 797         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 798                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 799                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 800
 801         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 802         return flowEntity;
 803     }
 804
 805     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 806
 807         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 808             routerName);
 809         List<MatchInfo> matches = new ArrayList<>();
 810         matches.add(MatchEthernetType.IPV4);
 811         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 812
 813         List<InstructionInfo> instructions = new ArrayList<>();
 814         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 815
 816         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 817         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 818             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 819             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 820
 821         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 822         return flowEntity;
 823     }
 824
 825     // TODO : Replace this with ITM Rpc once its available with full functionality
 826     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 827         TypedWriteTransaction<Configuration> confTx) {
 828
 829         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 830             dpnId, routerName);
 831         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 832         if (flowEntity == null) {
 833             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 834                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 835                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 836                     dpnId, routerName);
 837             return;
 838         }
 839         mdsalManager.addFlow(confTx, flowEntity);
 840
 841     }
 842
 843     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 844         List<MatchInfo> matches = new ArrayList<>();
 845         matches.add(MatchEthernetType.IPV4);
 846         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 847                 idManager, routerId, routerName);
 848         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 849         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 850         List<InstructionInfo> instructions = new ArrayList<>();
 851         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 852                 MetaDataUtil.METADATA_MASK_VRFID));
 853         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 854         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 855                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 856                 instructions);
 857         return flowEntity;
 858     }
 859
 860     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 861         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 862                 .FLOWID_SEPARATOR + routerID;
 863     }
 864
 865     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 866         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 867                 .FLOWID_SEPARATOR + routerID;
 868     }
 869
 870     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 871         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 872         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 873         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 874         List<BucketInfo> listBucketInfo = new ArrayList<>();
 875
 876         if (ifNamePrimary != null) {
 877             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 878             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 879                     interfaceManager, ifNamePrimary, routerId, true);
 880             if (listActionInfoPrimary.isEmpty()) {
 881                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 882                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 883                         ifNamePrimary);
 884             }
 885         } else {
 886             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 887                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 888         }
 889         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 890
 891         listBucketInfo.add(0, bucketPrimary);
 892         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 893     }
 894
 895     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 896                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 897         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 898         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 899         List<BucketInfo> listBucketInfo = new ArrayList<>();
 900
 901         if (ifNamePrimary != null) {
 902             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 903                     ifNamePrimary);
 904             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 905                     interfaceManager, ifNamePrimary, routerId, true);
 906             if (listActionInfoPrimary.isEmpty()) {
 907                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 908                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 909                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 910             }
 911         } else {
 912             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 913                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 914         }
 915         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 916
 917         listBucketInfo.add(0, bucketPrimary);
 918         return listBucketInfo;
 919     }
 920
 921     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 922         TypedWriteTransaction<Configuration> confTx) {
 923
 924        /*
 925         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 926         */
 927
 928         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 929
 930 /*
 931         List<BucketInfo> listBucketInfo = new ArrayList<>();
 932         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 933         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 934         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 935         listBucketInfo.add(0, bucketPrimary);
 936 */
 937
 938         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 939         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 940         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 941         installNaptPfibEntry(dpnId, routerId, confTx);
 942         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 943         if (networkId != null) {
 944             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 945             if (vpnUuid != null) {
 946                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 947                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 948                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 949                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 950                     if (vpnId != NatConstants.INVALID_ID) {
 951                         installNaptPfibEntry(dpnId, vpnId, null);
 952                     }
 953                     return Collections.emptyList();
 954                 });
 955             } else {
 956                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 957             }
 958         } else {
 959             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 960         }
 961     }
 962
 963     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
 964             @Nullable TypedWriteTransaction<Configuration> confTx) {
 965         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 966         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 967         if (confTx != null) {
 968             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 969         } else {
 970             mdsalManager.installFlow(naptPfibFlowEntity);
 971         }
 972     }
 973
 974     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 975
 976         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 977         List<MatchInfo> matches = new ArrayList<>();
 978         matches.add(MatchEthernetType.IPV4);
 979         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 980
 981         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 982         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 983         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
 984         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 985         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 986
 987         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
 988         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
 989             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 990             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
 991         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
 992         return flowEntity;
 993     }
 994
 995     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
 996         long routerId, String routerName, String externalIp) {
 997         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
 998             dpnId, routerId, externalIp);
 999         Uuid networkId = router.getNetworkId();
1000         if (networkId == null) {
1001             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1002             return;
1003         }
1004         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1005         if (vpnName == null) {
1006             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1007                     + "configuration {} in router {}", networkId, externalIp, routerId);
1008             return;
1009         }
1010         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1011             externalIp, networkId, router, confTx);
1012         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1013             dpnId, routerId, externalIp);
1014     }
1015
1016     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1017                                                 final long routerId, final String routerName, final String externalIp,
1018                                                 final Uuid extNetworkId, @Nullable final Routers router,
1019                                                 final TypedWriteTransaction<Configuration> confTx) {
1020         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1021                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1022         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1023         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1024         if (rd == null || rd.isEmpty()) {
1025             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1026             return;
1027         }
1028         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1029         if (extNwProvType == null) {
1030             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1031             return;
1032         }
1033         if (extNwProvType == ProviderTypes.VXLAN) {
1034             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1035                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1036             return;
1037         }
1038         //Generate VPN label for the external IP
1039         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1040             .setIpPrefix(externalIp).build();
1041         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1042
1043         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1044         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1045             if (result.isSuccessful()) {
1046                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1047                 GenerateVpnLabelOutput output = result.getResult();
1048                 final long label = output.getLabel();
1049
1050                 int externalIpInDsFlag = 0;
1051                 //Get IPMaps from the DB for the router ID
1052                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1053                 for (IpMap dbIpMap : dbIpMaps) {
1054                     String dbExternalIp = dbIpMap.getExternalIp();
1055                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1056                     if (dbExternalIp.contains(externalIp)) {
1057                         String dbInternalIp = dbIpMap.getInternalIp();
1058                         IpMapKey dbIpMapKey = dbIpMap.key();
1059                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1060                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1061                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1062                             .setExternalIp(dbExternalIp).setLabel(label).build();
1063                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1064                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1065                         externalIpInDsFlag++;
1066                     }
1067                 }
1068                 if (externalIpInDsFlag <= 0) {
1069                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1070                             + "Failed to update label {} for routerId {} in DS",
1071                             externalIp, label, routerId);
1072                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1073                         + " found in DS for router %s", label, externalIp, routerId);
1074                     return Futures.immediateFailedFuture(new Exception(errMsg));
1075                 }
1076                 //Inform BGP
1077                 long l3vni = 0;
1078                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1079                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1080                 }
1081                 Routers extRouter = router != null ? router :
1082                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1083                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1084                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1085                     RouteOrigin.STATIC, dpnId);
1086
1087                 //Install custom FIB routes
1088                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1089                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1090                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1091                         extNwProvType);
1092                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1093
1094                 //Install custom FIB routes - FIB table.
1095                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1096                         routerName, externalIp);
1097                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1098                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1099                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1100                 }
1101                 String externalVpn = vpnName;
1102                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1103                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1104                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1105                             externalSubnetId);
1106                     if (externalSubnet.isPresent()) {
1107                         externalVpn =  externalSubnetId.getValue();
1108                     }
1109                 }
1110                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1111                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1112                     .setVpnName(externalVpn)
1113                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1114                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1115                     .setInstruction(fibTableCustomInstructions).build();
1116                 return fibService.createFibEntry(input);
1117             } else {
1118                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1119                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1120                     externalIp, vpnName, result.getErrors());
1121                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1122             }
1123         }, MoreExecutors.directExecutor());
1124
1125         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1126
1127             @Override
1128             public void onFailure(@NonNull Throwable error) {
1129                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1130             }
1131
1132             @Override
1133             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1134                 if (result.isSuccessful()) {
1135                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1136                             externalIp);
1137                 } else {
1138                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1139                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1140                 }
1141             }
1142         }, MoreExecutors.directExecutor());
1143     }
1144
1145     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1146             String externalIp) {
1147         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1148         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1149         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1150                 externalIp, router);
1151         int instructionIndex = 0;
1152         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1153             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1154             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1155                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1156             instructionIndex++;
1157         }
1158
1159         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1160         return fibTableCustomInstructions;
1161     }
1162
1163     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1164         TypedWriteTransaction<Configuration> confTx) {
1165         List<MatchInfo> matches = new ArrayList<>();
1166         matches.add(MatchEthernetType.MPLS_UNICAST);
1167         matches.add(new MatchMplsLabel(serviceId));
1168
1169         List<Instruction> instructions = new ArrayList<>();
1170         List<ActionInfo> actionsInfos = new ArrayList<>();
1171         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1172         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1173         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1174         instructions.add(writeInstruction);
1175         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1176
1177         // Install the flow entry in L3_LFIB_TABLE
1178         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1179
1180         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1181             10, flowRef, 0, 0,
1182             COOKIE_VM_LFIB_TABLE, matches, instructions);
1183
1184         mdsalManager.addFlow(confTx, dpId, flowEntity);
1185
1186         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1187     }
1188
1189     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1190         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1191         ProviderTypes extNwProvType) {
1192
1193         List<MatchInfo> mkMatches = new ArrayList<>();
1194
1195         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1196             dpnId, serviceId, customInstructions);
1197
1198         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1199             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1200         } else {
1201             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1202         }
1203
1204         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1205             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1206                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1207                 String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
1208                 COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1209
1210         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1211     }
1212
1213     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1214         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1215             new RouterIdsKey(routerId)).build();
1216     }
1217
1218     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1219         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1220                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1221     }
1222
1223     @Override
1224     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1225         String routerName = original.getRouterName();
1226         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1227         if (routerId == NatConstants.INVALID_ID) {
1228             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1229             return;
1230         }
1231         // Check if its update on SNAT flag
1232         boolean originalSNATEnabled = original.isEnableSnat();
1233         boolean updatedSNATEnabled = update.isEnableSnat();
1234         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1235             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1236         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1237          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1238          */
1239         long bgpVpnId = NatConstants.INVALID_ID;
1240         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1241         if (bgpVpnUuid != null) {
1242             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1243         }
1244         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1245         final long finalBgpVpnId = bgpVpnId;
1246         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1247             List<ListenableFuture<Void>> futures = new ArrayList<>();
1248             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1249                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1250                     Uuid networkId = original.getNetworkId();
1251                     if (originalSNATEnabled != updatedSNATEnabled) {
1252                         if (originalSNATEnabled) {
1253                             if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1254                                 // Router has no interface attached
1255                                 return;
1256                             }
1257                             //SNAT disabled for the router
1258                             Uuid networkUuid = original.getNetworkId();
1259                             LOG.info("update : SNAT disabled for Router {}", routerName);
1260                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1261                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1262                                     removeFlowInvTx);
1263                         } else {
1264                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1265                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1266                                     true, writeFlowInvTx);
1267                             handleEnableSnat(update, routerId, dpnId, finalBgpVpnId, writeFlowInvTx);
1268                         }
1269                     }
1270                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1271                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1272                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1273                     }
1274
1275                     if (updatedSNATEnabled != originalSNATEnabled) {
1276                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1277                                 + "handleDisableSnat/handleEnableSnat");
1278                         return;
1279                     }
1280                     //Check if the Update is on External IPs
1281                     LOG.debug("update : Checking if this is update on External IPs");
1282                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1283                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1284
1285                     //Check if the External IPs are removed during the update.
1286                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1287                     removedExternalIps.removeAll(updatedExternalIps);
1288                     if (removedExternalIps.size() > 0) {
1289                         LOG.debug("update : Start processing of the External IPs removal during the update "
1290                                 + "operation");
1291                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1292                                 removedExternalIps, original.getExtGwMacAddress(),
1293                                 dpnId, networkId);
1294
1295                         for (String removedExternalIp : removedExternalIps) {
1296                 /*
1297                     1) Remove the mappings in the IntExt IP model which has external IP.
1298                     2) Remove the external IP in the ExternalCounter model.
1299                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1300                        least loaded external IP.
1301                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1302                     4) Increase the count of the allocated external IP by one.
1303                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1304                      i.e. the route for the external IP is absent.
1305                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1306                      the removed external IPs and also from the model.
1307                     7) Advertise to the BGP for removing the route for the removed external IPs.
1308                  */
1309
1310                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1311                             String externalIp = externalIpParts[0];
1312                             String externalIpPrefix = externalIpParts[1];
1313                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1314
1315                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1316                                     + "entries for removed external IP {}", externalIpAddrStr);
1317                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1318                             String vpnName = "";
1319                             if (vpnUuId != null) {
1320                                 vpnName = vpnUuId.getValue();
1321                             }
1322                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1323                                     update.getExtGwMacAddress(), removeFlowInvTx);
1324
1325                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1326                             //Get the internal IPs which are associated to the removed external IPs
1327                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1328                             List<String> removedInternalIps = new ArrayList<>();
1329                             for (IpMap ipMap : ipMaps) {
1330                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1331                                     removedInternalIps.add(ipMap.getInternalIp());
1332                                 }
1333                             }
1334
1335                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1336                             for (String removedInternalIp : removedInternalIps) {
1337                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1338                                                 + "router ID {} from the IntExtIP model",
1339                                         removedInternalIp, routerId);
1340                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1341                             }
1342
1343                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1344                                             + "router ID {} from the ExternalIpsCounter model.",
1345                                     externalIpAddrStr, routerId);
1346                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1347
1348                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1349                                     + "whose external IPs were removed.");
1350                             for (String removedInternalIp : removedInternalIps) {
1351                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1352                                         removedInternalIp, writeFlowInvTx);
1353                             }
1354                             LOG.debug("update : Remove the NAPT translation entries from "
1355                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1356                             //Get the internalIP and internal Port which were associated to the removed external IP.
1357                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1358                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1359                                     .builder(IntextIpPortMap.class)
1360                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1361                             Optional<IpPortMapping> ipPortMapping;
1362                             try {
1363                                 ipPortMapping = SingleTransactionDataBroker
1364                                             .syncReadOptional(dataBroker,
1365                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1366                             } catch (ReadFailedException e) {
1367                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1368                                 ipPortMapping = Optional.absent();
1369                             }
1370
1371                             if (ipPortMapping.isPresent()) {
1372                                 for (IntextIpProtocolType intextIpProtocolType :
1373                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1374                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1375                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1376                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1377                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1378                                             List<String> removedInternalIpPorts =
1379                                                     protoTypesIntIpPortsMap.get(protoType);
1380                                             if (removedInternalIpPorts != null) {
1381                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1382                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1383                                             } else {
1384                                                 removedInternalIpPorts = new ArrayList<>();
1385                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1386                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1387                                             }
1388                                         }
1389                                     }
1390                                 }
1391                             }
1392
1393                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1394                             // the removed external IP.
1395                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1396                                     protoTypesIntIpPortsMap.entrySet();
1397                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1398                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1399                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1400                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1401                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1402                                     // Remove the IP port map from the intext-ip-port-map model,
1403                                     // which were containing the removed external IP
1404                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1405                                             protocolType);
1406                                     //Remove the IP port incomint packer map.
1407                                     naptPacketInHandler.removeIncomingPacketMap(
1408                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1409                                     String[] removedInternalIpPortParts = removedInternalIpPort
1410                                             .split(NatConstants.COLON_SEPARATOR);
1411                                     if (removedInternalIpPortParts.length == 2) {
1412                                         String removedInternalIp = removedInternalIpPortParts[0];
1413                                         String removedInternalPort = removedInternalIpPortParts[1];
1414                                         List<String> removedInternalPortsList =
1415                                                 internalIpPortMap.get(removedInternalPort);
1416                                         if (removedInternalPortsList != null) {
1417                                             removedInternalPortsList.add(removedInternalPort);
1418                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1419                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1420                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1421                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1422                                             //Remove the NAPT translation entries from Outbound NAPT table
1423                                             naptEventHandler.removeNatFlows(dpnId,
1424                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1425                                                 routerId, removedInternalIp,
1426                                                 Integer.parseInt(removedInternalPort),
1427                                                 protocolType.getName());
1428                                             naptEventHandler.removeNatFlows(dpnId,
1429                                                 NwConstants.INBOUND_NAPT_TABLE,
1430                                                 routerId, removedInternalIp,
1431                                                 Integer.parseInt(removedInternalPort),
1432                                                 protocolType.getName());
1433                                         } else {
1434                                             removedInternalPortsList = new ArrayList<>();
1435                                             removedInternalPortsList.add(removedInternalPort);
1436                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1437                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1438                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1439                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1440                                             //Remove the NAPT translation entries from Outbound NAPT table
1441                                             naptEventHandler.removeNatFlows(dpnId,
1442                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1443                                                 routerId, removedInternalIp,
1444                                                 Integer.parseInt(removedInternalPort),
1445                                                 protocolType.getName());
1446                                             naptEventHandler.removeNatFlows(dpnId,
1447                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1448                                                 Integer.parseInt(removedInternalPort),
1449                                                 protocolType.getName());
1450                                         }
1451                                     }
1452                                 }
1453                             }
1454
1455                             // Delete the entry from SnatIntIpPortMap DS
1456                             Set<String> internalIps = internalIpPortMap.keySet();
1457                             for (String internalIp : internalIps) {
1458                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1459                                         + "model SnatIntIpPortMap", internalIp);
1460                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1461                             }
1462
1463                             naptManager.removeNaptPortPool(externalIp);
1464                         }
1465                         LOG.debug(
1466                                 "update : End processing of the External IPs removal during the update operation");
1467                     }
1468
1469                     //Check if the External IPs are added during the update.
1470                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1471                     addedExternalIps.removeAll(originalExternalIps);
1472                     if (addedExternalIps.size() != 0) {
1473                         LOG.debug("update : Start processing of the External IPs addition during the update "
1474                                 + "operation");
1475                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1476                                 update.getExtGwMacAddress(), dpnId,
1477                                 update.getNetworkId());
1478
1479                         for (String addedExternalIp : addedExternalIps) {
1480                 /*
1481                     1) Do nothing in the IntExtIp model.
1482                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1483                  */
1484                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1485                             String externalIp = externalIpParts[0];
1486                             String externalIpPrefix = externalIpParts[1];
1487                             String externalpStr = externalIp + "/" + externalIpPrefix;
1488                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1489                                             + "router ID {} in the ExternalIpsCounter model.",
1490                                     externalpStr, routerId);
1491                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1492                             subnetRegisterMapping(update, routerId);
1493                             LOG.info("update : Installing fib flow fo newly added Ips");
1494                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1495                         }
1496                         LOG.debug(
1497                                 "update : End processing of the External IPs addition during the update operation");
1498                     }
1499
1500                     //Check if its Update on subnets
1501                     LOG.debug("update : Checking if this is update on subnets");
1502                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1503                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1504                     Set<Uuid> addedSubnetIds =
1505                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1506                     if (originalSubnetIds != null) {
1507                         addedSubnetIds.removeAll(originalSubnetIds);
1508                     }
1509
1510                     //Check if the Subnet IDs are added during the update.
1511                     if (addedSubnetIds.size() != 0) {
1512                         LOG.debug(
1513                                 "update : Start processing of the Subnet IDs addition during the update operation");
1514                         for (Uuid addedSubnetId : addedSubnetIds) {
1515                 /*
1516                     1) Select the least loaded external IP for the subnet and store the mapping of the
1517                     subnet IP and the external IP in the IntExtIp model.
1518                     2) Increase the count of the selected external IP by one.
1519                     3) Advertise to the BGP if external IP is allocated for the first time for the
1520                     router i.e. the route for the external IP is absent.
1521                  */
1522                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1523                             if (subnetIp != null) {
1524                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1525                                         writeFlowInvTx);
1526                             }
1527                         }
1528                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1529                     }
1530
1531                     //Check if the Subnet IDs are removed during the update.
1532                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1533                     removedSubnetIds.removeAll(updatedSubnetIds);
1534                     if (removedSubnetIds.size() != 0) {
1535                         LOG.debug(
1536                                 "update : Start processing of the Subnet IDs removal during the update operation");
1537                         for (Uuid removedSubnetId : removedSubnetIds) {
1538                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1539                             if (subnetAddr != null) {
1540                     /*
1541                         1) Remove the subnet IP and the external IP in the IntExtIp map
1542                         2) Decrease the count of the coresponding external IP by one.
1543                         3) Advertise to the BGP for removing the routes of the corresponding external
1544                         IP if its not allocated to any other internal IP.
1545                     */
1546
1547                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1548                                         subnetAddr[0] + "/" + subnetAddr[1]);
1549                                 if (externalIp == null) {
1550                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1551                                             routerId, subnetAddr[0]);
1552                                     return;
1553                                 }
1554
1555                                 naptManager.updateCounter(routerId, externalIp, false);
1556                                 // Traverse entire model of external-ip counter whether external ip is not
1557                                 // used by any other internal ip in any router
1558                                 if (!isExternalIpAllocated(externalIp)) {
1559                                     LOG.debug("update : external ip is not allocated to any other "
1560                                             + "internal IP so proceeding to remove routes");
1561                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1562                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1563                                             removeFlowInvTx);
1564                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1565                                                     + "router {} with networkId {} and externalIp {}",
1566                                             dpnId, routerId, networkId, externalIp);
1567                                 }
1568
1569                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1570                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1571                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1572                             }
1573                         }
1574                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1575                     }
1576                 }));
1577             }));
1578             return futures;
1579         }, NatConstants.NAT_DJC_MAX_RETRIES);
1580     }
1581
1582     private boolean isExternalIpAllocated(String externalIp) {
1583         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1584         Optional<ExternalIpsCounter> externalCountersData;
1585         try {
1586             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1587                         LogicalDatastoreType.OPERATIONAL, id);
1588         } catch (ReadFailedException e) {
1589             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1590             externalCountersData = Optional.absent();
1591         }
1592         if (externalCountersData.isPresent()) {
1593             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1594             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1595                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1596                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1597                         if (externalIpCount.getCounter() != 0) {
1598                             return true;
1599                         }
1600                         break;
1601                     }
1602                 }
1603             }
1604         }
1605         return false;
1606     }
1607
1608     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1609             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1610         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1611         try {
1612             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1613             if (address instanceof Inet6Address) {
1614                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1615                 return;
1616             }
1617         } catch (UnknownHostException e) {
1618             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1619             return;
1620         }
1621         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1622         if (leastLoadedExtIpAddr != null) {
1623             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1624             String leastLoadedExtIp = externalIpParts[0];
1625             String leastLoadedExtIpPrefix = externalIpParts[1];
1626             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1627             subnetIp = subnetIpParts[0];
1628             String subnetIpPrefix = subnetIpParts[1];
1629             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1630             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1631                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1632                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1633             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1634
1635
1636             // Check if external IP is already assigned a route. (i.e. External IP is previously
1637             // allocated to any of the subnets)
1638             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1639             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1640             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1641             if (label != null) {
1642                 //update
1643                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1644                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1645                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1646                     label, internalIp, leastLoadedExtIpAddrStr);
1647                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1648                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1649                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1650                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1651                 return;
1652             }
1653
1654             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1655             // for the first time and hence not having a route.
1656             //Get the VPN Name using the network ID
1657             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1658             if (vpnName != null) {
1659                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1660                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1661                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1662                         + "added after gateway-set");
1663                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1664                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1665                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1666                         return;
1667                     }
1668                 }
1669                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1670                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1671                     writeFlowInvTx);
1672             }
1673         }
1674     }
1675
1676     @Nullable
1677     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1678         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1679         for (IpMap ipMap : ipMaps) {
1680             if (ipMap.getExternalIp().equals(externalIp)) {
1681                 if (ipMap.getLabel() != null) {
1682                     return ipMap.getLabel();
1683                 }
1684             }
1685         }
1686         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1687         return null;
1688     }
1689
1690     @Override
1691     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1692         LOG.trace("remove : Router delete method");
1693         /*
1694         ROUTER DELETE SCENARIO
1695         1) Get the router ID from the event.
1696         2) Build the cookie information from the router ID.
1697         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1698         4) Build the flow with the cookie value.
1699         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1700         6) Remove the flows from the other switches which points to the primary and secondary
1701          switches for the flows related the router ID.
1702         7) Get the list of external IP address maintained for the router ID.
1703         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1704         9) Withdraw the corresponding routes from the BGP.
1705          */
1706
1707         if (identifier == null || router == null) {
1708             LOG.error("remove : returning without processing since routers is null");
1709             return;
1710         }
1711
1712         String routerName = router.getRouterName();
1713         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1714             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1715                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1716                         routerName);
1717                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1718                 if (routerId == NatConstants.INVALID_ID) {
1719                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1720                             routerName);
1721                     return;
1722                 }
1723                 long bgpVpnId = NatConstants.INVALID_ID;
1724                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1725                 if (bgpVpnUuid != null) {
1726                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1727                 }
1728                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1729                         tx);
1730                 Uuid networkUuid = router.getNetworkId();
1731
1732                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1733                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1734                     // No NAPT switch for external router, probably because the router is not attached to
1735                     // any
1736                     // internal networks
1737                     LOG.debug(
1738                             "No NAPT switch for router {}, check if router is attached to any internal "
1739                                     + "network",
1740                             routerName);
1741                     return;
1742                 } else {
1743                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1744                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1745                             routerId, tx);
1746                 }
1747                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1748                     == NatConstants.INVALID_ID) {
1749                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1750                 }
1751             })), NatConstants.NAT_DJC_MAX_RETRIES);
1752     }
1753
1754     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1755                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1756                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1757         LOG.info("handleDisableSnat : Entry");
1758         String routerName = router.getRouterName();
1759         try {
1760             if (routerFlag) {
1761                 removeNaptSwitch(routerName);
1762             } else {
1763                 updateNaptSwitch(routerName, BigInteger.ZERO);
1764             }
1765
1766             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1767             naptManager.removeExternalCounter(routerId);
1768
1769             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1770             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1771                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1772                     + "router ID {} from RouterNaptSwitch model", routerId);
1773                 return;
1774             }
1775             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1776                     networkUuid);
1777             if (extNwProvType == null) {
1778                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1779                 return;
1780             }
1781             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1782             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1783                     externalSubnetList, removeFlowInvTx, extNwProvType);
1784             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1785             String externalSubnetVpn = null;
1786             for (Uuid externalSubnetId : externalSubnetList) {
1787                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1788                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1789                 if (externalSubnet.isPresent()) {
1790                     externalSubnetVpn =  externalSubnetId.getValue();
1791                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1792                             router.getExtGwMacAddress(), removeFlowInvTx);
1793                 }
1794             }
1795             if (externalSubnetVpn == null) {
1796                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1797                         router.getExtGwMacAddress(), removeFlowInvTx);
1798             }
1799             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1800             // for the router ID.
1801             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1802                     + "router ID {} in the DS", routerId);
1803             naptManager.removeMapping(routerId);
1804         } catch (InterruptedException | ExecutionException e) {
1805             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1806         }
1807         LOG.info("handleDisableSnat : Exit");
1808     }
1809
1810     // TODO Clean up the exception handling
1811     @SuppressWarnings("checkstyle:IllegalCatch")
1812     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1813                                              @NonNull Collection<String> externalIps,
1814                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1815         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1816                 + "with internet vpn {}", routerName, vpnId);
1817         try {
1818             BigInteger naptSwitchDpnId = null;
1819             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1820                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1821             Optional<RouterToNaptSwitch> rtrToNapt;
1822             try {
1823                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1824                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1825             } catch (ReadFailedException e) {
1826                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1827                 rtrToNapt = Optional.absent();
1828             }
1829             if (rtrToNapt.isPresent()) {
1830                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1831             }
1832             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1833
1834             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1835                     writeFlowInvTx);
1836             try {
1837                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1838                 if (extGwMacAddress != null) {
1839                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1840                             + "External Router ID {}", extGwMacAddress, routerId);
1841                 } else {
1842                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1843                             + "External Router ID {}", routerId);
1844                     return;
1845                 }
1846                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1847                         writeFlowInvTx);
1848             } catch (Exception ex) {
1849                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1850                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1851             }
1852             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1853                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1854             }
1855         } catch (InterruptedException | ExecutionException e) {
1856             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1857                     + "with internet vpn {}", routerName, vpnId, e);
1858         }
1859         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1860                 routerName, vpnId);
1861     }
1862
1863     // TODO Clean up the exception handling
1864     @SuppressWarnings("checkstyle:IllegalCatch")
1865     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1866         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1867             .setPrimarySwitchId(naptSwitchId).build();
1868         try {
1869             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1870                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1871         } catch (Exception ex) {
1872             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1873                 naptSwitchId, routerName);
1874         }
1875         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1876             naptSwitchId, routerName);
1877     }
1878
1879     protected void removeNaptSwitch(String routerName) {
1880         // Remove router and switch from model
1881         InstanceIdentifier<RouterToNaptSwitch> id =
1882             InstanceIdentifier.builder(NaptSwitches.class)
1883                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1884         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1885         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1886         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1887         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1888     }
1889
1890     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1891                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1892                                                 @NonNull Collection<String> externalIps,
1893                                                 Collection<Uuid> externalSubnetList,
1894                                                 TypedReadWriteTransaction<Configuration> confTx,
1895                                                 ProviderTypes extNwProvType)
1896             throws InterruptedException, ExecutionException {
1897
1898         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1899         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1900
1901         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1902         // traffic which comes from the  VMs of the NAPT switches)
1903         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1904         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1905
1906         LOG.info(
1907             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1908                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1909         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1910
1911         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1912         // traffic which comes from the VMs of the non NAPT switches)
1913         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1914                 elanManager, idManager, routerId, routerName);
1915         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1916         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1917         LOG.info(
1918             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1919                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1920         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1921
1922         //Remove the flow table 25->44 from NAPT Switch
1923         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1924             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1925         }
1926
1927         //Remove the Outbound flow entry which forwards the packet to FIB Table
1928         LOG.info(
1929             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1930                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1931
1932         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1933             NwConstants.IP_PROT_TCP);
1934         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1935             outboundTcpNatFlowRef);
1936         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1937
1938         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1939             NwConstants.IP_PROT_UDP);
1940         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1941             outboundUdpNatFlowRef);
1942         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1943
1944         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1945             NwConstants.IP_PROT_ICMP);
1946         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1947             icmpDropFlowRef);
1948         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1949         boolean lastRouterOnExternalNetwork =
1950             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1951         if (lastRouterOnExternalNetwork) {
1952             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1953         }
1954         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1955         // External Subnet Vpn Id.
1956         for (Uuid externalSubnetId : externalSubnetList) {
1957             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1958             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1959                 dataBroker, externalSubnetId, routerName, dpnId)) {
1960                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1961                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1962                     natPfibSubnetFlowRef);
1963                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1964                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1965                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1966                     subnetVpnId, dpnId);
1967             }
1968         }
1969
1970         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1971         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1972         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1973
1974         LOG.info(
1975             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1976             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1977         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1978
1979         if (lastRouterOnExternalNetwork) {
1980             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1981             // - This does not work since ext-routers is deleted already - no network info
1982             //Get the VPN ID from the ExternalNetworks model
1983             long vpnId = -1;
1984             if (vpnName == null || vpnName.isEmpty()) {
1985                 // ie called from router delete cases
1986                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1987                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1988                 if (vpnUuid != null) {
1989                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1990                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
1991                               + "or disableSNAT scenario", vpnId, networkId);
1992                 }
1993             } else {
1994                 // ie called from disassociate vpn case
1995                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
1996                     vpnName);
1997                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
1998                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
1999                     vpnId);
2000             }
2001
2002             if (vpnId != NatConstants.INVALID_ID) {
2003                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2004                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2005                 FlowEntity natPfibVpnFlowEntity =
2006                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2007                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2008                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2009                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2010             }
2011         }
2012
2013         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2014         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2015         if (ipPortMapping == null) {
2016             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2017             return;
2018         }
2019
2020         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2021             String protocol = intextIpProtocolType.getProtocol().name();
2022             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2023                 String ipPortInternal = ipPortMap.getIpPortInternal();
2024                 String[] ipPortParts = ipPortInternal.split(":");
2025                 if (ipPortParts.length != 2) {
2026                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2027                     return;
2028                 }
2029                 String internalIp = ipPortParts[0];
2030                 String internalPort = ipPortParts[1];
2031
2032                 //Build the flow for the outbound NAPT table
2033                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2034                     + NatConstants.COLON_SEPARATOR + internalPort);
2035                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2036                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2037                 FlowEntity outboundNaptFlowEntity =
2038                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2039
2040                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2041                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2042                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2043
2044                  //Build the flow for the inbound NAPT table
2045                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2046                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2047                 FlowEntity inboundNaptFlowEntity =
2048                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2049
2050                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2051                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2052                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2053             }
2054         }
2055     }
2056
2057     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2058                                                     @NonNull Collection<String> externalIps,
2059                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2060             throws ExecutionException, InterruptedException {
2061         long extVpnId = NatConstants.INVALID_ID;
2062         if (networkId != null) {
2063             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2064             if (vpnUuid != null) {
2065                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2066             } else {
2067                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2068             }
2069         } else {
2070             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2071             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2072         }
2073         if (extVpnId == NatConstants.INVALID_ID) {
2074             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2075             extVpnId = routerId;
2076         }
2077         for (String ip : externalIps) {
2078             String extIp = removeMaskFromIp(ip);
2079             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2080             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2081                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2082                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2083             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2084             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2085         }
2086     }
2087
2088     private String removeMaskFromIp(String ip) {
2089         if (ip != null && !ip.trim().isEmpty()) {
2090             return ip.split("/")[0];
2091         }
2092         return ip;
2093     }
2094
2095     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2096                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2097                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2098             throws ExecutionException, InterruptedException {
2099         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2100         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2101
2102         //Remove the NAPT PFIB TABLE entry
2103         long vpnId = -1;
2104         if (vpnName != null) {
2105             // ie called from disassociate vpn case
2106             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2107                     + "with vpnName {}", vpnName);
2108             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2109             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2110                     vpnId);
2111         }
2112
2113         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2114                 networkId, routerName, dpnId)) {
2115             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2116             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2117             FlowEntity natPfibVpnFlowEntity =
2118                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2119             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2120                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2121             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2122
2123             // Remove IP-PORT active NAPT entries and release port from IdManager
2124             // For the router ID get the internal IP , internal port and the corresponding
2125             // external IP and external Port.
2126             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2127             if (ipPortMapping == null) {
2128                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2129                 return;
2130             }
2131             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2132                 String protocol = intextIpProtocolType.getProtocol().name();
2133                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2134                     String ipPortInternal = ipPortMap.getIpPortInternal();
2135                     String[] ipPortParts = ipPortInternal.split(":");
2136                     if (ipPortParts.length != 2) {
2137                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2138                                 + "and port");
2139                         return;
2140                     }
2141                     String internalIp = ipPortParts[0];
2142                     String internalPort = ipPortParts[1];
2143
2144                     //Build the flow for the outbound NAPT table
2145                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2146                             + NatConstants.COLON_SEPARATOR + internalPort);
2147                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2148                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2149                     FlowEntity outboundNaptFlowEntity =
2150                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2151
2152                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2153                             + "active switch with the DPN ID {} and router ID {}",
2154                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2155                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2156
2157                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2158                     final String externalIp = ipPortExternal.getIpAddress();
2159
2160                     //Build the flow for the inbound NAPT table
2161                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2162                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2163                     FlowEntity inboundNaptFlowEntity =
2164                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2165
2166                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2167                             + "active active switch with the DPN ID {} and router ID {}",
2168                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2169                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2170
2171                     // Finally release port from idmanager
2172                     String internalIpPort = internalIp + ":" + internalPort;
2173                     naptManager.removePortFromPool(internalIpPort, externalIp);
2174
2175                     //Remove sessions from models
2176                     naptManager.removeIpPortMappingForRouterID(routerId);
2177                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2178                 }
2179             }
2180         } else {
2181             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2182         }
2183     }
2184
2185     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2186             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2187             throws ExecutionException, InterruptedException {
2188         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2189
2190         // Remove the flows from the other switches which points to the primary and secondary switches
2191         // for the flows related the router ID.
2192         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2193         if (allSwitchList.isEmpty()) {
2194             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2195             return;
2196         }
2197         for (BigInteger dpnId : allSwitchList) {
2198             if (!naptSwitchDpnId.equals(dpnId)) {
2199                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2200
2201                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2202                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2203                 FlowEntity preSnatFlowEntity =
2204                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2205
2206                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2207                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2208                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2209
2210                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2211                 long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2212                     NatUtil.getGroupIdKey(routerName));
2213                 if (groupId != NatConstants.INVALID_ID) {
2214                     LOG.info(
2215                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2216                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2217                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2218                 } else {
2219                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2220                 }
2221             }
2222         }
2223     }
2224
2225     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2226                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2227                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2228             throws ExecutionException, InterruptedException {
2229         //Withdraw the corresponding routes from the BGP.
2230         //Get the network ID using the router ID.
2231         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2232                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2233         if (networkUuid == null) {
2234             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2235             return;
2236         }
2237
2238         if (externalIps.isEmpty()) {
2239             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2240             return;
2241         }
2242
2243         if (vpnName == null) {
2244             //Get the VPN Name using the network ID
2245             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2246             if (vpnName == null) {
2247                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2248                     networkUuid, routerId);
2249                 return;
2250             }
2251         }
2252         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2253
2254         //Remove custom FIB routes
2255         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2256         for (String extIp : externalIps) {
2257             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2258         }
2259     }
2260
2261     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2262                                             final Uuid networkUuid, String extGwMacAddress,
2263                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2264             throws ExecutionException, InterruptedException {
2265         clearBgpRoutes(extIp, vpnName);
2266         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2267                 removeFlowInvTx);
2268     }
2269
2270     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2271                                              String vpnName, Uuid extNetworkId, long tempLabel,
2272                                              String gwMacAddress, boolean switchOver,
2273                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2274             throws ExecutionException, InterruptedException {
2275         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2276         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2277         if (routerName == null) {
2278             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2279             return;
2280         }
2281         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2282         if (extNwProvType == null) {
2283             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2284             return;
2285         }
2286         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2287          * external network provided type is VxLAN
2288          */
2289         if (extNwProvType == ProviderTypes.VXLAN) {
2290             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2291             );
2292             return;
2293         }
2294         if (tempLabel < 0) {
2295             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2296                     extIp, routerId);
2297             return;
2298         }
2299         final long label = tempLabel;
2300         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2301         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2302             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2303             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2304             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2305                     extRouter);
2306
2307             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2308                     externalSubnetId);
2309
2310             if (externalSubnet.isPresent()) {
2311                 vpnName =  externalSubnetId.getValue();
2312             }
2313         }
2314         final String externalVpn = vpnName;
2315         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2316                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2317                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2318         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2319
2320         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2321         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2322         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2323             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2324             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2325         }
2326         if (!switchOver) {
2327             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2328                 Futures.transformAsync(future, result -> {
2329                     //Release label
2330                     if (result.isSuccessful()) {
2331                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2332                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2333                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2334                         return vpnService.removeVpnLabel(labelInput);
2335                     } else {
2336                         String errMsg =
2337                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2338                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2339                         LOG.error(errMsg);
2340                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2341                     }
2342                 }, MoreExecutors.directExecutor());
2343
2344             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2345
2346                 @Override
2347                 public void onFailure(@NonNull Throwable error) {
2348                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2349                         + "got external ip {}", label, extIp, error);
2350                 }
2351
2352                 @Override
2353                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2354                     if (result.isSuccessful()) {
2355                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2356                             + "from VPN {}", externalIp, externalVpn);
2357                     } else {
2358                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2359                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2360                     }
2361                 }
2362             }, MoreExecutors.directExecutor());
2363         } else {
2364             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2365                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2366         }
2367     }
2368
2369     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2370                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2371                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2372             throws ExecutionException, InterruptedException {
2373         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2374         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2375         if (routerName == null) {
2376             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2377             return;
2378         }
2379         //Get the external network provider type from networkId
2380         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2381         if (extNwProvType == null) {
2382             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2383                     networkUuid);
2384             return;
2385         }
2386         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2387          *  external network provided type is VxLAN
2388          */
2389         if (extNwProvType == ProviderTypes.VXLAN) {
2390             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2391             return;
2392         }
2393         //Get IPMaps from the DB for the router ID
2394         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2395         if (dbIpMaps.isEmpty()) {
2396             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2397             return;
2398         }
2399
2400         long tempLabel = NatConstants.INVALID_ID;
2401         for (IpMap dbIpMap : dbIpMaps) {
2402             String dbExternalIp = dbIpMap.getExternalIp();
2403             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2404             //Select the IPMap, whose external IP is the IP for which FIB is installed
2405             if (extIp.equals(dbExternalIp)) {
2406                 tempLabel = dbIpMap.getLabel();
2407                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2408                     tempLabel, dbExternalIp, routerId);
2409                 break;
2410             }
2411         }
2412         if (tempLabel == NatConstants.INVALID_ID) {
2413             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2414                     extIp, routerId);
2415             return;
2416         }
2417
2418         final long label = tempLabel;
2419         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2420         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2421                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2422                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2423         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2424
2425         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2426         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2427         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2428             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2429             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2430         }
2431         if (!switchOver) {
2432             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2433                     Futures.transformAsync(future, result -> {
2434                         //Release label
2435                         if (result.isSuccessful()) {
2436                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2437                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2438                             return vpnService.removeVpnLabel(labelInput);
2439                         } else {
2440                             String errMsg =
2441                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2442                                             + "prefix %s Failed - %s",
2443                                             dpnId, externalIp, result.getErrors());
2444                             LOG.error(errMsg);
2445                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2446                         }
2447                     }, MoreExecutors.directExecutor());
2448
2449             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2450
2451                 @Override
2452                 public void onFailure(@NonNull Throwable error) {
2453                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2454                 }
2455
2456                 @Override
2457                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2458                     if (result.isSuccessful()) {
2459                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2460                             + "from VPN {}", externalIp, vpnName);
2461                     } else {
2462                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2463                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2464                     }
2465                 }
2466             }, MoreExecutors.directExecutor());
2467         } else {
2468             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2469                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2470         }
2471     }
2472
2473     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2474             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2475             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2476         //Withdraw the corresponding routes from the BGP.
2477         //Get the network ID using the router ID.
2478         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2479                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2480         if (networkUuid == null) {
2481             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2482             return;
2483         }
2484
2485         if (externalIps == null || externalIps.isEmpty()) {
2486             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2487             return;
2488         }
2489
2490         if (vpnName == null) {
2491             //Get the VPN Name using the network ID
2492             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2493             if (vpnName == null) {
2494                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2495                     networkUuid, routerId);
2496                 return;
2497             }
2498         }
2499         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2500
2501         //Remove custom FIB routes
2502         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2503         for (String extIp : externalIps) {
2504             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2505                     writeFlowInvTx);
2506         }
2507     }
2508
2509     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2510         //Inform BGP about the route removal
2511         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2512         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2513         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2514     }
2515
2516     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2517             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2518         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2519         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2520             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2521         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2522     }
2523
2524     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2525             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2526         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2527         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2528         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2529         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2530     }
2531
2532     /**
2533      * router association to vpn.
2534      *
2535      * @param routerName - Name of router
2536      * @param routerId - router id
2537      * @param bgpVpnName BGP VPN name
2538      */
2539     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2540             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2541         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2542         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2543             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2544
2545             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2546
2547             if (bgpVpnId != NatConstants.INVALID_ID) {
2548                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2549                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2550                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2551                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2552                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2553                     getRoutersIdentifier(bgpVpnId), rtrs);
2554
2555                 // Get the allocated Primary NAPT Switch for this router
2556                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2557                         routerId, bgpVpnId);
2558                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2559
2560                 // Get the group ID
2561                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2562                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2563                         extNwProvType);
2564             }
2565         }
2566     }
2567
2568     /**
2569      * router disassociation from vpn.
2570      *
2571      * @param routerName - Name of router
2572      * @param routerId - router id
2573      * @param bgpVpnName BGP VPN name
2574      */
2575     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2576             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2577         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2578         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2579             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2580             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2581
2582             // Get the allocated Primary NAPT Switch for this router
2583             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2584
2585             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2586             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2587
2588             // Get the group ID
2589             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2590             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2591                     writeFlowInvTx, extNwProvType);
2592         }
2593     }
2594
2595     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2596         InstanceIdentifier<Routers> routerInstanceIndentifier =
2597             InstanceIdentifier.builder(ExtRouters.class)
2598                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2599         try {
2600             Optional<Routers> routerData = SingleTransactionDataBroker
2601                     .syncReadOptional(dataBroker,
2602                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2603             return routerData.isPresent() && routerData.get().isEnableSnat();
2604         } catch (ReadFailedException e) {
2605             LOG.error("Failed to read data for router id {}", routerUuid, e);
2606             return false;
2607         }
2608     }
2609
2610     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2611         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2612
2613         long changedVpnId = bgpVpnId;
2614         String idType = "BGP VPN";
2615         if (bgpVpnId == NatConstants.INVALID_ID) {
2616             changedVpnId = routerId;
2617             idType = "router";
2618         }
2619
2620         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2621         if (switches.isEmpty()) {
2622             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2623             return;
2624         }
2625         for (BigInteger dpnId : switches) {
2626             // Update the BGP VPN ID in the SNAT miss entry to group
2627             if (!dpnId.equals(primarySwitchId)) {
2628                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2629                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2630                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2631                 long groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2632                     NatUtil.getGroupIdKey(routerName));
2633                 if (groupId != NatConstants.INVALID_ID) {
2634                     if (!isSnatCfgd) {
2635                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2636                     }
2637                     LOG.debug(
2638                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2639                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2640                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2641                         groupId, changedVpnId);
2642                     mdsalManager.addFlow(confTx, flowEntity);
2643                 } else {
2644                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2645                 }
2646             } else {
2647                 LOG.debug(
2648                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2649                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2650                 FlowEntity flowEntity =
2651                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2652                 mdsalManager.addFlow(confTx, flowEntity);
2653
2654                 LOG.debug(
2655                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2656                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2657                         idType, changedVpnId, primarySwitchId);
2658                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2659                         changedVpnId, confTx, extNwProvType);
2660
2661                 LOG.debug(
2662                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2663                                 + "which punts the packet to the controller in the Primary switch {}",
2664                         idType, changedVpnId, primarySwitchId);
2665                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2666
2667                 LOG.debug(
2668                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2669                                 + " outgoing packet to FIB Table in the Primary switch {}",
2670                         idType, changedVpnId, primarySwitchId);
2671                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2672
2673                 LOG.debug(
2674                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2675                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2676                                 + " {}", idType, changedVpnId, primarySwitchId);
2677                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2678
2679                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2680                         primarySwitchId);
2681                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2682                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2683                 if (vpnId != NatConstants.INVALID_ID) {
2684                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2685                 }
2686             }
2687         }
2688     }
2689
2690     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2691         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2692         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2693         if (ipPortMapping == null) {
2694             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2695             return;
2696         }
2697         // Get the External Gateway MAC Address
2698         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2699         if (extGwMacAddress != null) {
2700             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2701                     extGwMacAddress, routerId);
2702         } else {
2703             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2704                     routerId);
2705             return;
2706         }
2707         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2708             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2709                 String ipPortInternal = ipPortMap.getIpPortInternal();
2710                 String[] ipPortParts = ipPortInternal.split(":");
2711                 if (ipPortParts.length != 2) {
2712                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2713                     return;
2714                 }
2715                 String internalIp = ipPortParts[0];
2716                 String internalPort = ipPortParts[1];
2717                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2718                         internalIp, internalPort);
2719                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2720                 NAPTEntryEvent.Protocol protocol;
2721                 switch (protocolTypes) {
2722                     case TCP:
2723                         protocol = NAPTEntryEvent.Protocol.TCP;
2724                         break;
2725                     case UDP:
2726                         protocol = NAPTEntryEvent.Protocol.UDP;
2727                         break;
2728                     default:
2729                         protocol = NAPTEntryEvent.Protocol.TCP;
2730                 }
2731                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2732                 SessionAddress externalAddress =
2733                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2734                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2735                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2736                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2737                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2738                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2739             }
2740         }
2741     }
2742
2743     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2744                                                           long changedVpnId) {
2745
2746         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2747             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2748         List<MatchInfo> matches = new ArrayList<>();
2749         matches.add(MatchEthernetType.IPV4);
2750         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2751
2752         List<ActionInfo> actionsInfo = new ArrayList<>();
2753         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2754                 elanManager, idManager, changedVpnId, routerName);
2755         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2756         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2757                 actionsInfo);
2758         actionsInfo.add(new ActionGroup(groupId));
2759         List<InstructionInfo> instructions = new ArrayList<>();
2760         instructions.add(new InstructionApplyActions(actionsInfo));
2761         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2762         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2763             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2764             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2765
2766         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2767         return flowEntity;
2768     }
2769
2770     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2771                                                                         long changedVpnId) {
2772
2773         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2774                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2775         List<MatchInfo> matches = new ArrayList<>();
2776         matches.add(MatchEthernetType.IPV4);
2777         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2778
2779         List<InstructionInfo> instructions = new ArrayList<>();
2780         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2781
2782         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2783         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2784             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2785             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2786
2787         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2788         return flowEntity;
2789     }
2790
2791     // TODO : Replace this with ITM Rpc once its available with full functionality
2792     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2793         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2794
2795         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2796             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2797         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2798                 extNwProvType);
2799         mdsalManager.addFlow(confTx, flowEntity);
2800     }
2801
2802     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2803             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2804         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2805             dpId, routerName, changedVpnId);
2806         List<MatchInfo> matches = new ArrayList<>();
2807         matches.add(MatchEthernetType.IPV4);
2808
2809         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2810         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2811             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2812         }
2813         matches.add(new MatchTunnelId(tunnelId));
2814
2815         List<InstructionInfo> instructions = new ArrayList<>();
2816         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2817             MetaDataUtil.METADATA_MASK_VRFID));
2818         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2819         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2820         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2821         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2822             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2823             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2824         return flowEntity;
2825     }
2826
2827     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2828                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2829         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2830             dpnId, routerId, changedVpnId);
2831         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2832                 NwConstants.IP_PROT_TCP);
2833         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2834         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2835
2836         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2837                 NwConstants.IP_PROT_UDP);
2838         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2839         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2840
2841         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2842         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2843         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2844     }
2845
2846     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2847                                                            long changedVpnId, int protocol) {
2848         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2849             dpId, routerId, changedVpnId);
2850         BigInteger cookie = getCookieOutboundFlow(routerId);
2851         List<MatchInfo> matches = new ArrayList<>();
2852         matches.add(MatchEthernetType.IPV4);
2853         matches.add(new MatchIpProtocol((short)protocol));
2854         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2855
2856         List<InstructionInfo> instructions = new ArrayList<>();
2857         List<ActionInfo> actionsInfos = new ArrayList<>();
2858         actionsInfos.add(new ActionPuntToController());
2859         if (snatPuntTimeout != 0) {
2860             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2861         }
2862         instructions.add(new InstructionApplyActions(actionsInfos));
2863
2864         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2865         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2866             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2867         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2868         return flowEntity;
2869     }
2870
2871     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2872                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2873         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2874             dpnId, segmentId, changedVpnId);
2875         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2876         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2877     }
2878
2879     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2880
2881         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2882             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2883         List<MatchInfo> matches = new ArrayList<>();
2884         matches.add(MatchEthernetType.IPV4);
2885         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2886
2887         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2888         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2889         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2890         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2891         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2892
2893         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2894         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2895             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2896             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2897         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2898         return flowEntity;
2899     }
2900
2901     @Override
2902     protected ExternalRoutersListener getDataTreeChangeListener() {
2903         return ExternalRoutersListener.this;
2904     }
2905
2906     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2907                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2908         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2909                 routerName);
2910         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2911             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2912             if (subnetVpnId != -1) {
2913                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2914                     + "and vpnId {}", dpnId, subnetVpnId);
2915                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2916             }
2917         }
2918     }
2919 }