natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.netvirt.natservice.internal.NatUtil.requireNonNullElse;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import java.math.BigInteger;
  20 import java.net.Inet6Address;
  21 import java.net.InetAddress;
  22 import java.net.UnknownHostException;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Collections;
  27 import java.util.HashMap;
  28 import java.util.HashSet;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Set;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.Future;
  35 import javax.annotation.Nonnull;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  48 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  49 import org.opendaylight.genius.infra.TypedWriteTransaction;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.mdsalutil.ActionInfo;
  52 import org.opendaylight.genius.mdsalutil.BucketInfo;
  53 import org.opendaylight.genius.mdsalutil.FlowEntity;
  54 import org.opendaylight.genius.mdsalutil.GroupEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  70 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  73 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  74 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  79 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  80 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  81 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  82 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  83 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  84 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  85 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  86 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  87 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  88 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  91 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 150 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 151 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 152 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 153 import org.opendaylight.yangtools.yang.common.RpcResult;
 154 import org.slf4j.Logger;
 155 import org.slf4j.LoggerFactory;
 156
 157
 158 @Singleton
 159 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 160     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 161
 162     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 163     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 164
 165     private final DataBroker dataBroker;
 166     private final ManagedNewTransactionRunner txRunner;
 167     private final IMdsalApiManager mdsalManager;
 168     private final ItmRpcService itmManager;
 169     private final OdlInterfaceRpcService odlInterfaceRpcService;
 170     private final IdManagerService idManager;
 171     private final NaptManager naptManager;
 172     private final NAPTSwitchSelector naptSwitchSelector;
 173     private final IBgpManager bgpManager;
 174     private final VpnRpcService vpnService;
 175     private final FibRpcService fibService;
 176     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 177     private final NaptEventHandler naptEventHandler;
 178     private final NaptPacketInHandler naptPacketInHandler;
 179     private final IFibManager fibManager;
 180     private final IVpnManager vpnManager;
 181     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 182     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 183     private final NatMode natMode;
 184     private final INeutronVpnManager nvpnManager;
 185     private final IElanService elanManager;
 186     private final JobCoordinator coordinator;
 187     private final UpgradeState upgradeState;
 188     private final IInterfaceManager interfaceManager;
 189     private final NatOverVxlanUtil natOverVxlanUtil;
 190     private final int snatPuntTimeout;
 191
 192     @Inject
 193     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 194                                    final ItmRpcService itmManager,
 195                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 196                                    final IdManagerService idManager,
 197                                    final NaptManager naptManager,
 198                                    final NAPTSwitchSelector naptSwitchSelector,
 199                                    final IBgpManager bgpManager,
 200                                    final VpnRpcService vpnService,
 201                                    final FibRpcService fibService,
 202                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 203                                    final NaptEventHandler naptEventHandler,
 204                                    final NaptPacketInHandler naptPacketInHandler,
 205                                    final IFibManager fibManager,
 206                                    final IVpnManager vpnManager,
 207                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 208                                    final INeutronVpnManager nvpnManager,
 209                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 210                                    final NatserviceConfig config,
 211                                    final IElanService elanManager,
 212                                    final JobCoordinator coordinator,
 213                                    final UpgradeState upgradeState,
 214                                    final NatOverVxlanUtil natOverVxlanUtil,
 215                                    final IInterfaceManager interfaceManager) {
 216         super(Routers.class, ExternalRoutersListener.class);
 217         this.dataBroker = dataBroker;
 218         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 219         this.mdsalManager = mdsalManager;
 220         this.itmManager = itmManager;
 221         this.odlInterfaceRpcService = odlInterfaceRpcService;
 222         this.idManager = idManager;
 223         this.naptManager = naptManager;
 224         this.naptSwitchSelector = naptSwitchSelector;
 225         this.bgpManager = bgpManager;
 226         this.vpnService = vpnService;
 227         this.fibService = fibService;
 228         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 229         this.naptEventHandler = naptEventHandler;
 230         this.naptPacketInHandler = naptPacketInHandler;
 231         this.fibManager = fibManager;
 232         this.vpnManager = vpnManager;
 233         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 234         this.nvpnManager = nvpnManager;
 235         this.elanManager = elanManager;
 236         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 237         this.coordinator = coordinator;
 238         this.upgradeState = upgradeState;
 239         this.interfaceManager = interfaceManager;
 240         this.natOverVxlanUtil = natOverVxlanUtil;
 241         if (config != null) {
 242             this.natMode = config.getNatMode();
 243             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 244         } else {
 245             this.natMode = NatMode.Controller;
 246             this.snatPuntTimeout = 0;
 247         }
 248     }
 249
 250     @Override
 251     @PostConstruct
 252     public void init() {
 253         LOG.info("{} init", getClass().getSimpleName());
 254         // This class handles ExternalRouters for Controller SNAT mode.
 255         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 256         if (natMode == NatMode.Controller) {
 257             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 258             NatUtil.createGroupIdPool(idManager);
 259         }
 260     }
 261
 262     @Override
 263     protected InstanceIdentifier<Routers> getWildCardPath() {
 264         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 265     }
 266
 267     @Override
 268     // TODO Clean up the exception handling
 269     @SuppressWarnings("checkstyle:IllegalCatch")
 270     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 271         // Populate the router-id-name container
 272         String routerName = routers.getRouterName();
 273         LOG.info("add : external router event for {}", routerName);
 274         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 275         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 276         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 277         try {
 278             if (routers.isEnableSnat()) {
 279                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 280                     () -> Collections.singletonList(
 281                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 282                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 283                             long bgpVpnId = NatConstants.INVALID_ID;
 284                             if (bgpVpnUuid != null) {
 285                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 286                             }
 287                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 288                             // Allocate Primary Napt Switch for this router
 289                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 290                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 291                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 292                             }
 293                         }
 294                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 295             } else {
 296                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 297             }
 298         } catch (Exception ex) {
 299             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 300                     routerName, ex);
 301         }
 302     }
 303
 304     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 305                                  TypedWriteTransaction<Configuration> confTx) {
 306         String routerName = routers.getRouterName();
 307         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 308
 309         naptManager.initialiseExternalCounter(routers, routerId);
 310         subnetRegisterMapping(routers, routerId);
 311
 312         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 313             primarySwitchId, routerName);
 314
 315         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 316                 routers.getNetworkId());
 317         if (extNwProvType == null) {
 318             LOG.error("handleEnableSnat : External Network Provider Type missing");
 319             return;
 320         }
 321
 322         if (bgpVpnId != NatConstants.INVALID_ID) {
 323             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 324                     extNwProvType);
 325         } else {
 326             // write metadata and punt
 327             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 328             // Now install entries in SNAT tables to point to Primary for each router
 329             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 330             for (BigInteger dpnId : switches) {
 331                 // Handle switches and NAPT switches separately
 332                 if (!dpnId.equals(primarySwitchId)) {
 333                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 334                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 335                 } else {
 336                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 337                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 338                 }
 339             }
 340         }
 341
 342         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 343         if (externalIps.isEmpty()) {
 344             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 345             return;
 346         } else {
 347             for (String externalIpAddrPrefix : externalIps) {
 348                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 349                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 350                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 351                 );
 352             }
 353         }
 354         LOG.debug("handleEnableSnat : Exit");
 355     }
 356
 357     private BigInteger getPrimaryNaptSwitch(String routerName) {
 358         // Allocate Primary Napt Switch for this router
 359         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 360         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 361             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 362                 primarySwitchId, routerName);
 363             return primarySwitchId;
 364         }
 365         // Allocated an id from VNI pool for the Router.
 366         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 367         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 368         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 369
 370         return primarySwitchId;
 371     }
 372
 373     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 374                                                      TypedWriteTransaction<Configuration> confTx) {
 375         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 376         if (extVpnId == NatConstants.INVALID_ID) {
 377             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 378             extVpnId = routerId;
 379         }
 380         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 381         if (externalIps.isEmpty()) {
 382             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 383                 dpnId, extVpnId);
 384             return;
 385         }
 386         for (String ip : externalIps) {
 387             Uuid subnetId = getSubnetIdForFixedIp(ip);
 388             if (subnetId != null) {
 389                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 390                 if (subnetVpnId != NatConstants.INVALID_ID) {
 391                     extVpnId = subnetVpnId;
 392                 }
 393                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 394                     dpnId, extVpnId, subnetId);
 395                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 396                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 397             }
 398         }
 399     }
 400
 401     @Nullable
 402     private Uuid getSubnetIdForFixedIp(String ip) {
 403         if (ip != null) {
 404             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 405             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 406             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 407         }
 408         LOG.error("getSubnetIdForFixedIp : ip is null");
 409         return null;
 410     }
 411
 412     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 413         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 414         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 415         int counter = 0;
 416         int extIpCounter = externalIps.size();
 417         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 418                 counter, extIpCounter);
 419         for (Uuid subnet : requireNonNullElse(routerEntry.getSubnetIds(), Collections.<Uuid>emptyList())) {
 420             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 421             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 422                 .builder(Subnetmaps.class)
 423                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 424                 .build();
 425             Optional<Subnetmap> sn;
 426             try {
 427                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 428                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 429             } catch (ReadFailedException e) {
 430                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 431                 sn = Optional.absent();
 432             }
 433             if (sn.isPresent()) {
 434                 // subnets
 435                 Subnetmap subnetmapEntry = sn.get();
 436                 String subnetString = subnetmapEntry.getSubnetIp();
 437                 String[] subnetSplit = subnetString.split("/");
 438                 String subnetIp = subnetSplit[0];
 439                 try {
 440                     InetAddress address = InetAddress.getByName(subnetIp);
 441                     if (address instanceof Inet6Address) {
 442                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 443                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 444                         continue;
 445                     }
 446                 } catch (UnknownHostException e) {
 447                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 448                     return;
 449                 }
 450                 String subnetPrefix = "0";
 451                 if (subnetSplit.length == 2) {
 452                     subnetPrefix = subnetSplit[1];
 453                 }
 454                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 455                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 456                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 457                 //externalIps
 458                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 459                         counter, extIpCounter);
 460                 if (extIpCounter != 0) {
 461                     if (counter < extIpCounter) {
 462                         String[] ipSplit = externalIps.get(counter).split("/");
 463                         String externalIp = ipSplit[0];
 464                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 465                         if (ipSplit.length == 2) {
 466                             extPrefix = ipSplit[1];
 467                         }
 468                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 469                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 470                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 471                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 472                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 473                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 474                     } else {
 475                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 476                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 477                         String[] ipSplit = externalIps.get(counter).split("/");
 478                         String externalIp = ipSplit[0];
 479                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 480                         if (ipSplit.length == 2) {
 481                             extPrefix = ipSplit[1];
 482                         }
 483                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 484                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 485                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 486                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 487                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 488                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 489                             externalIp, extPrefix);
 490                     }
 491                 }
 492                 counter++;
 493                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 494             } else {
 495                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 496             }
 497         }
 498     }
 499
 500     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 501             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 502             throws ExecutionException, InterruptedException {
 503         //Check if BGP VPN exists. If exists then invoke the new method.
 504         if (bgpVpnId != NatConstants.INVALID_ID) {
 505             if (bgpVpnUuid != null) {
 506                 String bgpVpnName = bgpVpnUuid.getValue();
 507                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 508                     bgpVpnId, bgpVpnName);
 509                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 510                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 511                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 512             }
 513             if (create) {
 514                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 515             } else {
 516                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 517             }
 518             return;
 519         }
 520
 521         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 522         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 523     }
 524
 525     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 526             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 527         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 528         if (switches.isEmpty()) {
 529             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 530             return;
 531         }
 532         if (routerId == NatConstants.INVALID_ID) {
 533             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 534                     + "default NAT route in FIB", routerName);
 535             return;
 536         }
 537         for (BigInteger dpnId : switches) {
 538             if (create) {
 539                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 540                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 541                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 542             } else {
 543                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 544                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 545                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 546             }
 547         }
 548     }
 549
 550     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 551         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 552         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 553         if (dpnIds.isEmpty()) {
 554             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 555                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 556             return;
 557         }
 558         for (BigInteger dpnId : dpnIds) {
 559             if (bgpVpnId != NatConstants.INVALID_ID) {
 560                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 561                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 562                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 563             } else {
 564                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 565                     + "in dpn {} for the internal vpn", routerId, dpnId);
 566                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 567             }
 568         }
 569     }
 570
 571     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 572             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 573             throws ExecutionException, InterruptedException {
 574         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 575         if (dpnIds.isEmpty()) {
 576             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 577                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 578             return;
 579         }
 580         for (BigInteger dpnId : dpnIds) {
 581             if (bgpVpnId != NatConstants.INVALID_ID) {
 582                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 583                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 584                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 585             } else {
 586                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 587                     + "in dpn {} for the internal vpn", routerId, dpnId);
 588                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 589             }
 590         }
 591     }
 592
 593     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 594         TypedWriteTransaction<Configuration> confTx) {
 595         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 596         if (routerId != NatConstants.INVALID_ID) {
 597             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 598                     primarySwitchId, routerId);
 599             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 600         } else {
 601             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 602                 + "createAndInstallMissEntry", routerName);
 603         }
 604     }
 605
 606     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 607         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 608                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 609     }
 610
 611     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 612         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 613                 .FLOWID_SEPARATOR + vpnId;
 614     }
 615
 616     public BigInteger getCookieOutboundFlow(long routerId) {
 617         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 618             BigInteger.valueOf(routerId));
 619     }
 620
 621     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 622         long l4SrcPortField;
 623         long l4DstPortField;
 624         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 625
 626         if (protocol == NwConstants.IP_PROT_TCP) {
 627             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 628             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 629         } else {
 630             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 631             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 632         }
 633         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 634                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 635                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 636                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 637                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 638                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 639                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 640                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 641                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 642                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 643                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 644                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 645                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 646                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 647                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 648
 649         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 650                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 651     }
 652
 653     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 654         List<MatchInfo> matches = new ArrayList<>();
 655         matches.add(MatchEthernetType.IPV4);
 656         matches.add(MatchIpProtocol.ICMP);
 657         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 658
 659         List<ActionInfo> actionInfos = new ArrayList<>();
 660         actionInfos.add(new ActionDrop());
 661
 662         List<InstructionInfo> instructions = new ArrayList<>();
 663         instructions.add(new InstructionApplyActions(actionInfos));
 664
 665         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 666                 NwConstants.IP_PROT_ICMP);
 667         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 668                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 669                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 670         return flow;
 671     }
 672
 673     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 674         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 675         BigInteger cookie = getCookieOutboundFlow(routerId);
 676         List<MatchInfo> matches = new ArrayList<>();
 677         matches.add(MatchEthernetType.IPV4);
 678         matches.add(new MatchIpProtocol((short)protocol));
 679         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 680
 681         List<InstructionInfo> instructions = new ArrayList<>();
 682         List<ActionInfo> actionsInfos = new ArrayList<>();
 683         actionsInfos.add(new ActionPuntToController());
 684         if (snatPuntTimeout != 0) {
 685             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 686         }
 687         instructions.add(new InstructionApplyActions(actionsInfos));
 688
 689         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 690         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 691             5, flowRef, 0, 0,
 692             cookie, matches, instructions);
 693         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 694         return flowEntity;
 695     }
 696
 697     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 698         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 699         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 700         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 701         mdsalManager.addFlow(confTx, tcpflowEntity);
 702
 703         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 704         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 705         mdsalManager.addFlow(confTx, udpflowEntity);
 706
 707         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 708         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 709         mdsalManager.addFlow(confTx, icmpDropFlow);
 710     }
 711
 712     @Nullable
 713     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 714         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 715         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 716         try {
 717             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 718                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 719                     .setSourceDpid(srcDpId)
 720                     .setDestinationDpid(dstDpId)
 721                     .setTunnelType(tunType)
 722                     .build());
 723             rpcResult = result.get();
 724             if (!rpcResult.isSuccessful()) {
 725                 tunType = TunnelTypeGre.class;
 726                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 727                     .setSourceDpid(srcDpId)
 728                     .setDestinationDpid(dstDpId)
 729                     .setTunnelType(tunType)
 730                     .build());
 731                 rpcResult = result.get();
 732                 if (!rpcResult.isSuccessful()) {
 733                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 734                             rpcResult.getErrors());
 735                 } else {
 736                     return rpcResult.getResult().getInterfaceName();
 737                 }
 738                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 739                         rpcResult.getErrors());
 740             } else {
 741                 return rpcResult.getResult().getInterfaceName();
 742             }
 743         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 744             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 745                     + "between {} and {}", srcDpId, dstDpId, e);
 746         }
 747
 748         return null;
 749     }
 750
 751     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 752         TypedWriteTransaction<Configuration> confTx) {
 753
 754         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 755         // Install miss entry pointing to group
 756         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 757         mdsalManager.addFlow(confTx, flowEntity);
 758     }
 759
 760     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 761             String routerName, long routerId) {
 762         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 763             dpnId, bucketInfo.get(0));
 764         // Install the select group
 765         long groupId = createGroupId(getGroupIdKey(routerName));
 766         GroupEntity groupEntity =
 767             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 768         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 769         mdsalManager.syncInstallGroup(groupEntity);
 770         // Install miss entry pointing to group
 771         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 772         if (flowEntity == null) {
 773             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 774                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 775                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 776             return;
 777         }
 778         mdsalManager.installFlow(flowEntity);
 779     }
 780
 781     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 782         long groupId = createGroupId(getGroupIdKey(routerName));
 783         GroupEntity groupEntity =
 784             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 785         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 786         mdsalManager.syncInstallGroup(groupEntity);
 787         return groupId;
 788     }
 789
 790     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 791         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 792                 dpId, routerName, groupId);
 793         List<MatchInfo> matches = new ArrayList<>();
 794         matches.add(MatchEthernetType.IPV4);
 795         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 796
 797         List<ActionInfo> actionsInfo = new ArrayList<>();
 798         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 799                 routerId, routerName);
 800         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 801         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 802         actionsInfo.add(new ActionGroup(groupId));
 803         List<InstructionInfo> instructions = new ArrayList<>();
 804         instructions.add(new InstructionApplyActions(actionsInfo));
 805         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 806         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 807                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 808                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 809
 810         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 811         return flowEntity;
 812     }
 813
 814     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 815
 816         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 817             routerName);
 818         List<MatchInfo> matches = new ArrayList<>();
 819         matches.add(MatchEthernetType.IPV4);
 820         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 821
 822         List<InstructionInfo> instructions = new ArrayList<>();
 823         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 824
 825         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 826         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 827             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 828             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 829
 830         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 831         return flowEntity;
 832     }
 833
 834     // TODO : Replace this with ITM Rpc once its available with full functionality
 835     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 836         TypedWriteTransaction<Configuration> confTx) {
 837
 838         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 839             dpnId, routerName);
 840         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 841         if (flowEntity == null) {
 842             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 843                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 844                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 845                     dpnId, routerName);
 846             return;
 847         }
 848         mdsalManager.addFlow(confTx, flowEntity);
 849
 850     }
 851
 852     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 853         List<MatchInfo> matches = new ArrayList<>();
 854         matches.add(MatchEthernetType.IPV4);
 855         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 856                 idManager, routerId, routerName);
 857         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 858         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 859         List<InstructionInfo> instructions = new ArrayList<>();
 860         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 861                 MetaDataUtil.METADATA_MASK_VRFID));
 862         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 863         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 864                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 865                 instructions);
 866         return flowEntity;
 867     }
 868
 869     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 870         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 871                 .FLOWID_SEPARATOR + routerID;
 872     }
 873
 874     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 875         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 876                 .FLOWID_SEPARATOR + routerID;
 877     }
 878
 879     private String getGroupIdKey(String routerName) {
 880         return "snatmiss." + routerName;
 881     }
 882
 883     protected long createGroupId(String groupIdKey) {
 884         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 885             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 886             .build();
 887         try {
 888             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 889             RpcResult<AllocateIdOutput> rpcResult = result.get();
 890             return rpcResult.getResult().getIdValue();
 891         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 892             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 893         }
 894         return 0;
 895     }
 896
 897     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 898         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 899         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 900         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 901         List<BucketInfo> listBucketInfo = new ArrayList<>();
 902
 903         if (ifNamePrimary != null) {
 904             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 905             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 906                     interfaceManager, ifNamePrimary, routerId, true);
 907             if (listActionInfoPrimary.isEmpty()) {
 908                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 909                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 910                         ifNamePrimary);
 911             }
 912         } else {
 913             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 914                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 915         }
 916         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 917
 918         listBucketInfo.add(0, bucketPrimary);
 919         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 920     }
 921
 922     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 923                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 924         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 925         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 926         List<BucketInfo> listBucketInfo = new ArrayList<>();
 927
 928         if (ifNamePrimary != null) {
 929             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 930                     ifNamePrimary);
 931             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 932                     interfaceManager, ifNamePrimary, routerId, true);
 933             if (listActionInfoPrimary.isEmpty()) {
 934                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 935                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 936                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 937             }
 938         } else {
 939             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 940                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 941         }
 942         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 943
 944         listBucketInfo.add(0, bucketPrimary);
 945         return listBucketInfo;
 946     }
 947
 948     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 949         TypedWriteTransaction<Configuration> confTx) {
 950
 951        /*
 952         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 953         */
 954
 955         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 956
 957 /*
 958         List<BucketInfo> listBucketInfo = new ArrayList<>();
 959         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 960         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 961         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 962         listBucketInfo.add(0, bucketPrimary);
 963 */
 964
 965         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 966         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 967         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 968         installNaptPfibEntry(dpnId, routerId, confTx);
 969         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 970         if (networkId != null) {
 971             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 972             if (vpnUuid != null) {
 973                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 974                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 975                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 976                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 977                     if (vpnId != NatConstants.INVALID_ID) {
 978                         installNaptPfibEntry(dpnId, vpnId, null);
 979                     }
 980                     return Collections.emptyList();
 981                 });
 982             } else {
 983                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 984             }
 985         } else {
 986             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 987         }
 988     }
 989
 990     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 991         List<BucketInfo> listBucketInfo = new ArrayList<>();
 992         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 993         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 994         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 995         listBucketInfo.add(0, bucketPrimary);
 996         return listBucketInfo;
 997     }
 998
 999     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
1000             @Nullable TypedWriteTransaction<Configuration> confTx) {
1001         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1002         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1003         if (confTx != null) {
1004             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1005         } else {
1006             mdsalManager.installFlow(naptPfibFlowEntity);
1007         }
1008     }
1009
1010     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1011
1012         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1013         List<MatchInfo> matches = new ArrayList<>();
1014         matches.add(MatchEthernetType.IPV4);
1015         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1016
1017         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1018         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1019         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1020         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1021         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1022
1023         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1024         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1025             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1026             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1027         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1028         return flowEntity;
1029     }
1030
1031     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1032         long routerId, String routerName, String externalIp) {
1033         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1034             dpnId, routerId, externalIp);
1035         Uuid networkId = router.getNetworkId();
1036         if (networkId == null) {
1037             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1038             return;
1039         }
1040         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1041         if (vpnName == null) {
1042             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1043                     + "configuration {} in router {}", networkId, externalIp, routerId);
1044             return;
1045         }
1046         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1047             externalIp, networkId, router, confTx);
1048         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1049             dpnId, routerId, externalIp);
1050     }
1051
1052     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1053                                                 final long routerId, final String routerName, final String externalIp,
1054                                                 final Uuid extNetworkId, @Nullable final Routers router,
1055                                                 final TypedWriteTransaction<Configuration> confTx) {
1056         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1057                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1058         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1059         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1060         if (rd == null || rd.isEmpty()) {
1061             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1062             return;
1063         }
1064         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1065         if (extNwProvType == null) {
1066             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1067             return;
1068         }
1069         if (extNwProvType == ProviderTypes.VXLAN) {
1070             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1071                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1072             return;
1073         }
1074         //Generate VPN label for the external IP
1075         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1076             .setIpPrefix(externalIp).build();
1077         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1078
1079         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1080         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1081             if (result.isSuccessful()) {
1082                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1083                 GenerateVpnLabelOutput output = result.getResult();
1084                 final long label = output.getLabel();
1085
1086                 int externalIpInDsFlag = 0;
1087                 //Get IPMaps from the DB for the router ID
1088                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1089                 for (IpMap dbIpMap : dbIpMaps) {
1090                     String dbExternalIp = dbIpMap.getExternalIp();
1091                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1092                     if (dbExternalIp.contains(externalIp)) {
1093                         String dbInternalIp = dbIpMap.getInternalIp();
1094                         IpMapKey dbIpMapKey = dbIpMap.key();
1095                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1096                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1097                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1098                             .setExternalIp(dbExternalIp).setLabel(label).build();
1099                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1100                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1101                         externalIpInDsFlag++;
1102                     }
1103                 }
1104                 if (externalIpInDsFlag <= 0) {
1105                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1106                             + "Failed to update label {} for routerId {} in DS",
1107                             externalIp, label, routerId);
1108                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1109                         + " found in DS for router %s", label, externalIp, routerId);
1110                     return Futures.immediateFailedFuture(new Exception(errMsg));
1111                 }
1112                 //Inform BGP
1113                 long l3vni = 0;
1114                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1115                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1116                 }
1117                 Routers extRouter = router != null ? router :
1118                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1119                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1120                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1121                     RouteOrigin.STATIC, dpnId);
1122
1123                 //Install custom FIB routes
1124                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1125                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1126                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1127                         extNwProvType);
1128                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1129
1130                 //Install custom FIB routes - FIB table.
1131                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1132                         routerName, externalIp);
1133                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1134                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1135                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1136                 }
1137                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1138                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1139                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1140                 String externalVpn = externalSubnet.isPresent() ? externalSubnetId.getValue() : vpnName;
1141                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1142                     .setVpnName(externalVpn)
1143                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1144                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1145                     .setInstruction(fibTableCustomInstructions).build();
1146                 return fibService.createFibEntry(input);
1147             } else {
1148                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1149                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1150                     externalIp, vpnName, result.getErrors());
1151                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1152             }
1153         }, MoreExecutors.directExecutor());
1154
1155         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1156
1157             @Override
1158             public void onFailure(@Nonnull Throwable error) {
1159                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1160             }
1161
1162             @Override
1163             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1164                 if (result.isSuccessful()) {
1165                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1166                             externalIp);
1167                 } else {
1168                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1169                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1170                 }
1171             }
1172         }, MoreExecutors.directExecutor());
1173     }
1174
1175     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1176             String externalIp) {
1177         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1178         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1179         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1180                 externalIp, router);
1181         int instructionIndex = 0;
1182         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1183             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1184             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1185                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1186             instructionIndex++;
1187         }
1188
1189         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1190         return fibTableCustomInstructions;
1191     }
1192
1193     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1194         TypedWriteTransaction<Configuration> confTx) {
1195         List<MatchInfo> matches = new ArrayList<>();
1196         matches.add(MatchEthernetType.MPLS_UNICAST);
1197         matches.add(new MatchMplsLabel(serviceId));
1198
1199         List<Instruction> instructions = new ArrayList<>();
1200         List<ActionInfo> actionsInfos = new ArrayList<>();
1201         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1202         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1203         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1204         instructions.add(writeInstruction);
1205         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1206
1207         // Install the flow entry in L3_LFIB_TABLE
1208         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1209
1210         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1211             10, flowRef, 0, 0,
1212             COOKIE_VM_LFIB_TABLE, matches, instructions);
1213
1214         mdsalManager.addFlow(confTx, dpId, flowEntity);
1215
1216         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1217     }
1218
1219     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1220         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1221         ProviderTypes extNwProvType) {
1222
1223         List<MatchInfo> mkMatches = new ArrayList<>();
1224
1225         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1226             dpnId, serviceId, customInstructions);
1227
1228         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1229             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1230         } else {
1231             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1232         }
1233
1234         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1235             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1236             String.format("%s:%d", "TST Flow Entry ", serviceId),
1237             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1238
1239         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1240     }
1241
1242     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1243         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1244             new RouterIdsKey(routerId)).build();
1245     }
1246
1247     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1248         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1249                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1250     }
1251
1252     @Override
1253     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1254         String routerName = original.getRouterName();
1255         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1256         if (routerId == NatConstants.INVALID_ID) {
1257             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1258             return;
1259         }
1260         // Check if its update on SNAT flag
1261         boolean originalSNATEnabled = original.isEnableSnat();
1262         boolean updatedSNATEnabled = update.isEnableSnat();
1263         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1264             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1265         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1266          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1267          */
1268         long bgpVpnId = NatConstants.INVALID_ID;
1269         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1270         if (bgpVpnUuid != null) {
1271             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1272         }
1273         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1274         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1275             // Router has no interface attached
1276             return;
1277         }
1278         final long finalBgpVpnId = bgpVpnId;
1279         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1280             List<ListenableFuture<Void>> futures = new ArrayList<>();
1281             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1282                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1283                     Uuid networkId = original.getNetworkId();
1284                     if (originalSNATEnabled != updatedSNATEnabled) {
1285                         if (originalSNATEnabled) {
1286                             //SNAT disabled for the router
1287                             Uuid networkUuid = original.getNetworkId();
1288                             LOG.info("update : SNAT disabled for Router {}", routerName);
1289                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1290                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1291                                     removeFlowInvTx);
1292                         } else {
1293                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1294                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1295                                     true, writeFlowInvTx);
1296                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1297                         }
1298                     }
1299                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1300                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1301                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1302                     }
1303
1304                     //Check if the Update is on External IPs
1305                     LOG.debug("update : Checking if this is update on External IPs");
1306                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1307                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1308
1309                     //Check if the External IPs are added during the update.
1310                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1311                     addedExternalIps.removeAll(originalExternalIps);
1312                     if (addedExternalIps.size() != 0) {
1313                         LOG.debug("update : Start processing of the External IPs addition during the update "
1314                                 + "operation");
1315                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1316                                 update.getExtGwMacAddress(), dpnId,
1317                                 update.getNetworkId());
1318
1319                         for (String addedExternalIp : addedExternalIps) {
1320                 /*
1321                     1) Do nothing in the IntExtIp model.
1322                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1323                  */
1324                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1325                             String externalIp = externalIpParts[0];
1326                             String externalIpPrefix = externalIpParts[1];
1327                             String externalpStr = externalIp + "/" + externalIpPrefix;
1328                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1329                                             + "router ID {} in the ExternalIpsCounter model.",
1330                                     externalpStr, routerId);
1331                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1332                         }
1333                         LOG.debug(
1334                                 "update : End processing of the External IPs addition during the update operation");
1335                     }
1336
1337                     //Check if the External IPs are removed during the update.
1338                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1339                     removedExternalIps.removeAll(updatedExternalIps);
1340                     if (removedExternalIps.size() > 0) {
1341                         LOG.debug("update : Start processing of the External IPs removal during the update "
1342                                 + "operation");
1343                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1344                                 removedExternalIps, original.getExtGwMacAddress(),
1345                                 dpnId, networkId);
1346
1347                         for (String removedExternalIp : removedExternalIps) {
1348                 /*
1349                     1) Remove the mappings in the IntExt IP model which has external IP.
1350                     2) Remove the external IP in the ExternalCounter model.
1351                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1352                        least loaded external IP.
1353                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1354                     4) Increase the count of the allocated external IP by one.
1355                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1356                      i.e. the route for the external IP is absent.
1357                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1358                      the removed external IPs and also from the model.
1359                     7) Advertise to the BGP for removing the route for the removed external IPs.
1360                  */
1361
1362                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1363                             String externalIp = externalIpParts[0];
1364                             String externalIpPrefix = externalIpParts[1];
1365                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1366
1367                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1368                                     + "entries for removed external IP {}", externalIpAddrStr);
1369                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1370                             String vpnName = "";
1371                             if (vpnUuId != null) {
1372                                 vpnName = vpnUuId.getValue();
1373                             }
1374                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1375                                     update.getExtGwMacAddress(), removeFlowInvTx);
1376
1377                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1378                             //Get the internal IPs which are associated to the removed external IPs
1379                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1380                             List<String> removedInternalIps = new ArrayList<>();
1381                             for (IpMap ipMap : ipMaps) {
1382                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1383                                     removedInternalIps.add(ipMap.getInternalIp());
1384                                 }
1385                             }
1386
1387                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1388                             for (String removedInternalIp : removedInternalIps) {
1389                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1390                                                 + "router ID {} from the IntExtIP model",
1391                                         removedInternalIp, routerId);
1392                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1393                             }
1394
1395                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1396                                             + "router ID {} from the ExternalIpsCounter model.",
1397                                     externalIpAddrStr, routerId);
1398                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1399
1400                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1401                                     + "whose external IPs were removed.");
1402                             for (String removedInternalIp : removedInternalIps) {
1403                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1404                                         removedInternalIp, writeFlowInvTx);
1405                             }
1406                             LOG.debug("update : Remove the NAPT translation entries from "
1407                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1408                             //Get the internalIP and internal Port which were associated to the removed external IP.
1409                             List<Integer> externalPorts = new ArrayList<>();
1410                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1411                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1412                                     .builder(IntextIpPortMap.class)
1413                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1414                             Optional<IpPortMapping> ipPortMapping;
1415                             try {
1416                                 ipPortMapping = SingleTransactionDataBroker
1417                                             .syncReadOptional(dataBroker,
1418                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1419                             } catch (ReadFailedException e) {
1420                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1421                                 ipPortMapping = Optional.absent();
1422                             }
1423
1424                             if (ipPortMapping.isPresent()) {
1425                                 for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(
1426                                         ipPortMapping.get().getIntextIpProtocolType(),
1427                                         Collections.<IntextIpProtocolType>emptyList())) {
1428                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1429                                     for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
1430                                             Collections.<IpPortMap>emptyList())) {
1431                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1432                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1433                                             externalPorts.add(ipPortExternal.getPortNum());
1434                                             List<String> removedInternalIpPorts =
1435                                                     protoTypesIntIpPortsMap.get(protoType);
1436                                             if (removedInternalIpPorts != null) {
1437                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1438                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1439                                             } else {
1440                                                 removedInternalIpPorts = new ArrayList<>();
1441                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1442                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1443                                             }
1444                                         }
1445                                     }
1446                                 }
1447                             }
1448
1449                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1450                             // the removed external IP.
1451                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1452                                     protoTypesIntIpPortsMap.entrySet();
1453                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1454                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1455                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1456                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1457                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1458                                     // Remove the IP port map from the intext-ip-port-map model,
1459                                     // which were containing the removed external IP
1460                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1461                                             protocolType);
1462                                     //Remove the IP port incomint packer map.
1463                                     naptPacketInHandler.removeIncomingPacketMap(
1464                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1465                                     String[] removedInternalIpPortParts = removedInternalIpPort
1466                                             .split(NatConstants.COLON_SEPARATOR);
1467                                     if (removedInternalIpPortParts.length == 2) {
1468                                         String removedInternalIp = removedInternalIpPortParts[0];
1469                                         String removedInternalPort = removedInternalIpPortParts[1];
1470                                         List<String> removedInternalPortsList =
1471                                                 internalIpPortMap.get(removedInternalPort);
1472                                         if (removedInternalPortsList != null) {
1473                                             removedInternalPortsList.add(removedInternalPort);
1474                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1475                                         } else {
1476                                             removedInternalPortsList = new ArrayList<>();
1477                                             removedInternalPortsList.add(removedInternalPort);
1478                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1479                                         }
1480                                     }
1481                                 }
1482                             }
1483
1484                             // Delete the entry from SnatIntIpPortMap DS
1485                             Set<String> internalIps = internalIpPortMap.keySet();
1486                             for (String internalIp : internalIps) {
1487                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1488                                         + "model SnatIntIpPortMap", internalIp);
1489                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1490                             }
1491
1492                             naptManager.removeNaptPortPool(externalIp);
1493
1494                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1495                                     + "the removed external IP {}", externalIp);
1496                             for (Integer externalPort : externalPorts) {
1497                                 //Remove the NAPT translation entries from Inbound NAPT table
1498                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1499                                         routerId, externalIp, externalPort);
1500                             }
1501
1502                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1503                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1504                                 String internalIp = internalIpPort.getKey();
1505                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1506                                         + "for the removed internal IP {}", internalIp);
1507                                 List<String> internalPorts = internalIpPort.getValue();
1508                                 for (String internalPort : internalPorts) {
1509                                     //Remove the NAPT translation entries from Outbound NAPT table
1510                                     naptPacketInHandler.removeIncomingPacketMap(
1511                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1512                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1513                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1514                                             routerId, internalIp, Integer.parseInt(internalPort));
1515                                 }
1516                             }
1517                         }
1518                         LOG.debug(
1519                                 "update : End processing of the External IPs removal during the update operation");
1520                     }
1521
1522                     //Check if its Update on subnets
1523                     LOG.debug("update : Checking if this is update on subnets");
1524                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1525                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1526                     Set<Uuid> addedSubnetIds =
1527                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1528                     if (originalSubnetIds != null) {
1529                         addedSubnetIds.removeAll(originalSubnetIds);
1530                     }
1531
1532                     //Check if the Subnet IDs are added during the update.
1533                     if (addedSubnetIds.size() != 0) {
1534                         LOG.debug(
1535                                 "update : Start processing of the Subnet IDs addition during the update operation");
1536                         for (Uuid addedSubnetId : addedSubnetIds) {
1537                 /*
1538                     1) Select the least loaded external IP for the subnet and store the mapping of the
1539                     subnet IP and the external IP in the IntExtIp model.
1540                     2) Increase the count of the selected external IP by one.
1541                     3) Advertise to the BGP if external IP is allocated for the first time for the
1542                     router i.e. the route for the external IP is absent.
1543                  */
1544                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1545                             if (subnetIp != null) {
1546                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1547                                         writeFlowInvTx);
1548                             }
1549                         }
1550                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1551                     }
1552
1553                     //Check if the Subnet IDs are removed during the update.
1554                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1555                     removedSubnetIds.removeAll(updatedSubnetIds);
1556                     if (removedSubnetIds.size() != 0) {
1557                         LOG.debug(
1558                                 "update : Start processing of the Subnet IDs removal during the update operation");
1559                         for (Uuid removedSubnetId : removedSubnetIds) {
1560                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1561                             if (subnetAddr != null) {
1562                     /*
1563                         1) Remove the subnet IP and the external IP in the IntExtIp map
1564                         2) Decrease the count of the coresponding external IP by one.
1565                         3) Advertise to the BGP for removing the routes of the corresponding external
1566                         IP if its not allocated to any other internal IP.
1567                     */
1568
1569                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1570                                         subnetAddr[0] + "/" + subnetAddr[1]);
1571                                 if (externalIp == null) {
1572                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1573                                             routerId, subnetAddr[0]);
1574                                     return;
1575                                 }
1576
1577                                 naptManager.updateCounter(routerId, externalIp, false);
1578                                 // Traverse entire model of external-ip counter whether external ip is not
1579                                 // used by any other internal ip in any router
1580                                 if (!isExternalIpAllocated(externalIp)) {
1581                                     LOG.debug("update : external ip is not allocated to any other "
1582                                             + "internal IP so proceeding to remove routes");
1583                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1584                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1585                                             removeFlowInvTx);
1586                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1587                                                     + "router {} with networkId {} and externalIp {}",
1588                                             dpnId, routerId, networkId, externalIp);
1589                                 }
1590
1591                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1592                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1593                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1594                             }
1595                         }
1596                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1597                     }
1598                 }));
1599             }));
1600             return futures;
1601         }, NatConstants.NAT_DJC_MAX_RETRIES);
1602     }
1603
1604     private boolean isExternalIpAllocated(String externalIp) {
1605         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1606         Optional<ExternalIpsCounter> externalCountersData;
1607         try {
1608             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1609                         LogicalDatastoreType.OPERATIONAL, id);
1610         } catch (ReadFailedException e) {
1611             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1612             externalCountersData = Optional.absent();
1613         }
1614         if (externalCountersData.isPresent()) {
1615             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1616             for (ExternalCounters ext : requireNonNullElse(externalIpsCounters.getExternalCounters(),
1617                     Collections.<ExternalCounters>emptyList())) {
1618                 for (ExternalIpCounter externalIpCount : requireNonNullElse(ext.getExternalIpCounter(),
1619                         Collections.<ExternalIpCounter>emptyList())) {
1620                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1621                         if (externalIpCount.getCounter() != 0) {
1622                             return true;
1623                         }
1624                         break;
1625                     }
1626                 }
1627             }
1628         }
1629         return false;
1630     }
1631
1632     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1633             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1634         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1635         try {
1636             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1637             if (address instanceof Inet6Address) {
1638                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1639                 return;
1640             }
1641         } catch (UnknownHostException e) {
1642             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1643             return;
1644         }
1645         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1646         if (leastLoadedExtIpAddr != null) {
1647             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1648             String leastLoadedExtIp = externalIpParts[0];
1649             String leastLoadedExtIpPrefix = externalIpParts[1];
1650             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1651             subnetIp = subnetIpParts[0];
1652             String subnetIpPrefix = subnetIpParts[1];
1653             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1654             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1655                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1656                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1657             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1658
1659
1660             // Check if external IP is already assigned a route. (i.e. External IP is previously
1661             // allocated to any of the subnets)
1662             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1663             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1664             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1665             if (label != null) {
1666                 //update
1667                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1668                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1669                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1670                     label, internalIp, leastLoadedExtIpAddrStr);
1671                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1672                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1673                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1674                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1675                 return;
1676             }
1677
1678             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1679             // for the first time and hence not having a route.
1680             //Get the VPN Name using the network ID
1681             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1682             if (vpnName != null) {
1683                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1684                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1685                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1686                         + "added after gateway-set");
1687                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1688                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1689                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1690                         return;
1691                     }
1692                 }
1693                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1694                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1695                     writeFlowInvTx);
1696             }
1697         }
1698     }
1699
1700     @Nullable
1701     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1702         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1703         for (IpMap ipMap : ipMaps) {
1704             if (ipMap.getExternalIp().equals(externalIp)) {
1705                 if (ipMap.getLabel() != null) {
1706                     return ipMap.getLabel();
1707                 }
1708             }
1709         }
1710         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1711         return null;
1712     }
1713
1714     @Override
1715     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1716         LOG.trace("remove : Router delete method");
1717         /*
1718         ROUTER DELETE SCENARIO
1719         1) Get the router ID from the event.
1720         2) Build the cookie information from the router ID.
1721         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1722         4) Build the flow with the cookie value.
1723         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1724         6) Remove the flows from the other switches which points to the primary and secondary
1725          switches for the flows related the router ID.
1726         7) Get the list of external IP address maintained for the router ID.
1727         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1728         9) Withdraw the corresponding routes from the BGP.
1729          */
1730
1731         if (identifier == null || router == null) {
1732             LOG.error("remove : returning without processing since routers is null");
1733             return;
1734         }
1735
1736         String routerName = router.getRouterName();
1737         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1738             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1739                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1740                         routerName);
1741                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1742                 if (routerId == NatConstants.INVALID_ID) {
1743                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1744                             routerName);
1745                     return;
1746                 }
1747                 long bgpVpnId = NatConstants.INVALID_ID;
1748                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1749                 if (bgpVpnUuid != null) {
1750                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1751                 }
1752                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1753                         tx);
1754                 Uuid networkUuid = router.getNetworkId();
1755
1756                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1757                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1758                     // No NAPT switch for external router, probably because the router is not attached to
1759                     // any
1760                     // internal networks
1761                     LOG.debug(
1762                             "No NAPT switch for router {}, check if router is attached to any internal "
1763                                     + "network",
1764                             routerName);
1765                     return;
1766                 } else {
1767                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1768                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1769                             routerId, tx);
1770                 }
1771                 natOverVxlanUtil.releaseVNI(routerName);
1772             })), NatConstants.NAT_DJC_MAX_RETRIES);
1773     }
1774
1775     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1776                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1777                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1778         LOG.info("handleDisableSnat : Entry");
1779         String routerName = router.getRouterName();
1780         try {
1781             if (routerFlag) {
1782                 removeNaptSwitch(routerName);
1783             } else {
1784                 updateNaptSwitch(routerName, BigInteger.ZERO);
1785             }
1786
1787             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1788             naptManager.removeExternalCounter(routerId);
1789
1790             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1791             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1792                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1793                     + "router ID {} from RouterNaptSwitch model", routerId);
1794                 return;
1795             }
1796             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1797                     networkUuid);
1798             if (extNwProvType == null) {
1799                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1800                 return;
1801             }
1802             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1803             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1804                     externalSubnetList, removeFlowInvTx, extNwProvType);
1805             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1806             String externalSubnetVpn = null;
1807             for (Uuid externalSubnetId : externalSubnetList) {
1808                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1809                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1810                 if (externalSubnet.isPresent()) {
1811                     externalSubnetVpn =  externalSubnetId.getValue();
1812                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1813                             router.getExtGwMacAddress(), removeFlowInvTx);
1814                 }
1815             }
1816             if (externalSubnetVpn == null) {
1817                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1818                         router.getExtGwMacAddress(), removeFlowInvTx);
1819             }
1820             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1821             // for the router ID.
1822             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1823                     + "router ID {} in the DS", routerId);
1824             naptManager.removeMapping(routerId);
1825         } catch (InterruptedException | ExecutionException e) {
1826             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1827         }
1828         LOG.info("handleDisableSnat : Exit");
1829     }
1830
1831     // TODO Clean up the exception handling
1832     @SuppressWarnings("checkstyle:IllegalCatch")
1833     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1834                                              @Nonnull Collection<String> externalIps,
1835                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1836         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1837                 + "with internet vpn {}", routerName, vpnId);
1838         try {
1839             BigInteger naptSwitchDpnId = null;
1840             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1841                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1842             Optional<RouterToNaptSwitch> rtrToNapt;
1843             try {
1844                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1845                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1846             } catch (ReadFailedException e) {
1847                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1848                 rtrToNapt = Optional.absent();
1849             }
1850             if (rtrToNapt.isPresent()) {
1851                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1852             }
1853             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1854
1855             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1856                     writeFlowInvTx);
1857             try {
1858                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1859                 if (extGwMacAddress != null) {
1860                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1861                             + "External Router ID {}", extGwMacAddress, routerId);
1862                 } else {
1863                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1864                             + "External Router ID {}", routerId);
1865                     return;
1866                 }
1867                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1868                         writeFlowInvTx);
1869             } catch (Exception ex) {
1870                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1871                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1872             }
1873             natOverVxlanUtil.releaseVNI(vpnId);
1874         } catch (InterruptedException | ExecutionException e) {
1875             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1876                     + "with internet vpn {}", routerName, vpnId, e);
1877         }
1878         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1879                 routerName, vpnId);
1880     }
1881
1882     // TODO Clean up the exception handling
1883     @SuppressWarnings("checkstyle:IllegalCatch")
1884     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1885         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1886             .setPrimarySwitchId(naptSwitchId).build();
1887         try {
1888             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1889                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1890         } catch (Exception ex) {
1891             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1892                 naptSwitchId, routerName);
1893         }
1894         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1895             naptSwitchId, routerName);
1896     }
1897
1898     protected void removeNaptSwitch(String routerName) {
1899         // Remove router and switch from model
1900         InstanceIdentifier<RouterToNaptSwitch> id =
1901             InstanceIdentifier.builder(NaptSwitches.class)
1902                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1903         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1904         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1905         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1906         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1907     }
1908
1909     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1910                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1911                                                 @Nonnull Collection<String> externalIps,
1912                                                 Collection<Uuid> externalSubnetList,
1913                                                 TypedReadWriteTransaction<Configuration> confTx,
1914                                                 ProviderTypes extNwProvType)
1915             throws InterruptedException, ExecutionException {
1916
1917         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1918         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1919
1920         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1921         // traffic which comes from the  VMs of the NAPT switches)
1922         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1923         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1924
1925         LOG.info(
1926             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1927                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1928         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1929
1930         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1931         // traffic which comes from the VMs of the non NAPT switches)
1932         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1933                 elanManager, idManager, routerId, routerName);
1934         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1935         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1936         LOG.info(
1937             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1938                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1939         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1940
1941         //Remove the flow table 25->44 from NAPT Switch
1942         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1943             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1944         }
1945
1946         //Remove the Outbound flow entry which forwards the packet to FIB Table
1947         LOG.info(
1948             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1949                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1950
1951         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1952             NwConstants.IP_PROT_TCP);
1953         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1954             outboundTcpNatFlowRef);
1955         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1956
1957         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1958             NwConstants.IP_PROT_UDP);
1959         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1960             outboundUdpNatFlowRef);
1961         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1962
1963         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1964             NwConstants.IP_PROT_ICMP);
1965         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1966             icmpDropFlowRef);
1967         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1968         boolean lastRouterOnExternalNetwork =
1969             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1970         if (lastRouterOnExternalNetwork) {
1971             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1972         }
1973         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1974         // External Subnet Vpn Id.
1975         for (Uuid externalSubnetId : externalSubnetList) {
1976             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1977             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1978                 dataBroker, externalSubnetId, routerName, dpnId)) {
1979                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1980                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1981                     natPfibSubnetFlowRef);
1982                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1983                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1984                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1985                     subnetVpnId, dpnId);
1986             }
1987         }
1988
1989         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1990         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1991         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1992
1993         LOG.info(
1994             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1995             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1996         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1997
1998         if (lastRouterOnExternalNetwork) {
1999             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2000             // - This does not work since ext-routers is deleted already - no network info
2001             //Get the VPN ID from the ExternalNetworks model
2002             long vpnId = -1;
2003             if (vpnName == null || vpnName.isEmpty()) {
2004                 // ie called from router delete cases
2005                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2006                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2007                 if (vpnUuid != null) {
2008                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2009                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2010                               + "or disableSNAT scenario", vpnId, networkId);
2011                 }
2012             } else {
2013                 // ie called from disassociate vpn case
2014                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2015                     vpnName);
2016                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2017                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2018                     vpnId);
2019             }
2020
2021             if (vpnId != NatConstants.INVALID_ID) {
2022                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2023                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2024                 FlowEntity natPfibVpnFlowEntity =
2025                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2026                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2027                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2028                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2029             }
2030         }
2031
2032         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2033         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2034         if (ipPortMapping == null) {
2035             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2036             return;
2037         }
2038
2039         for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2040                 Collections.<IntextIpProtocolType>emptyList())) {
2041             for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2042                     Collections.<IpPortMap>emptyList())) {
2043                 String ipPortInternal = ipPortMap.getIpPortInternal();
2044                 String[] ipPortParts = ipPortInternal.split(":");
2045                 if (ipPortParts.length != 2) {
2046                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2047                     return;
2048                 }
2049                 String internalIp = ipPortParts[0];
2050                 String internalPort = ipPortParts[1];
2051
2052                 //Build the flow for the outbound NAPT table
2053                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2054                     + NatConstants.COLON_SEPARATOR + internalPort);
2055                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2056                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2057                 FlowEntity outboundNaptFlowEntity =
2058                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2059
2060                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2061                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2062                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2063
2064                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2065                 String externalIp = ipPortExternal.getIpAddress();
2066                 int externalPort = ipPortExternal.getPortNum();
2067
2068                 //Build the flow for the inbound NAPT table
2069                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2070                     String.valueOf(routerId), externalIp, externalPort);
2071                 FlowEntity inboundNaptFlowEntity =
2072                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2073
2074                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2075                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2076                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2077             }
2078         }
2079     }
2080
2081     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2082                                                     @Nonnull Collection<String> externalIps,
2083                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2084             throws ExecutionException, InterruptedException {
2085         long extVpnId = NatConstants.INVALID_ID;
2086         if (networkId != null) {
2087             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2088             if (vpnUuid != null) {
2089                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2090             } else {
2091                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2092             }
2093         } else {
2094             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2095             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2096         }
2097         if (extVpnId == NatConstants.INVALID_ID) {
2098             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2099             extVpnId = routerId;
2100         }
2101         for (String ip : externalIps) {
2102             String extIp = removeMaskFromIp(ip);
2103             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2104             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2105                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2106                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2107             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2108             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2109         }
2110     }
2111
2112     private String removeMaskFromIp(String ip) {
2113         if (ip != null && !ip.trim().isEmpty()) {
2114             return ip.split("/")[0];
2115         }
2116         return ip;
2117     }
2118
2119     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2120                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2121                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2122             throws ExecutionException, InterruptedException {
2123         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2124         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2125
2126         //Remove the NAPT PFIB TABLE entry
2127         long vpnId = -1;
2128         if (vpnName != null) {
2129             // ie called from disassociate vpn case
2130             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2131                     + "with vpnName {}", vpnName);
2132             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2133             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2134                     vpnId);
2135         }
2136
2137         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2138                 networkId, routerName, dpnId)) {
2139             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2140             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2141             FlowEntity natPfibVpnFlowEntity =
2142                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2143             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2144                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2145             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2146
2147             // Remove IP-PORT active NAPT entries and release port from IdManager
2148             // For the router ID get the internal IP , internal port and the corresponding
2149             // external IP and external Port.
2150             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2151             if (ipPortMapping == null) {
2152                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2153                 return;
2154             }
2155             for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2156                     Collections.<IntextIpProtocolType>emptyList())) {
2157                 for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2158                         Collections.<IpPortMap>emptyList())) {
2159                     String ipPortInternal = ipPortMap.getIpPortInternal();
2160                     String[] ipPortParts = ipPortInternal.split(":");
2161                     if (ipPortParts.length != 2) {
2162                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2163                                 + "and port");
2164                         return;
2165                     }
2166                     String internalIp = ipPortParts[0];
2167                     String internalPort = ipPortParts[1];
2168
2169                     //Build the flow for the outbound NAPT table
2170                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2171                             + NatConstants.COLON_SEPARATOR + internalPort);
2172                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2173                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2174                     FlowEntity outboundNaptFlowEntity =
2175                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2176
2177                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2178                             + "active switch with the DPN ID {} and router ID {}",
2179                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2180                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2181
2182                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2183                     String externalIp = ipPortExternal.getIpAddress();
2184                     int externalPort = ipPortExternal.getPortNum();
2185
2186                     //Build the flow for the inbound NAPT table
2187                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2188                         String.valueOf(routerId), externalIp, externalPort);
2189                     FlowEntity inboundNaptFlowEntity =
2190                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2191
2192                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2193                             + "active active switch with the DPN ID {} and router ID {}",
2194                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2195                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2196
2197                     // Finally release port from idmanager
2198                     String internalIpPort = internalIp + ":" + internalPort;
2199                     naptManager.removePortFromPool(internalIpPort, externalIp);
2200
2201                     //Remove sessions from models
2202                     naptManager.removeIpPortMappingForRouterID(routerId);
2203                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2204                 }
2205             }
2206         } else {
2207             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2208         }
2209     }
2210
2211     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2212             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2213             throws ExecutionException, InterruptedException {
2214         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2215
2216         // Remove the flows from the other switches which points to the primary and secondary switches
2217         // for the flows related the router ID.
2218         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2219         if (allSwitchList.isEmpty()) {
2220             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2221             return;
2222         }
2223         for (BigInteger dpnId : allSwitchList) {
2224             if (!naptSwitchDpnId.equals(dpnId)) {
2225                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2226
2227                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2228                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2229                 FlowEntity preSnatFlowEntity =
2230                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2231
2232                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2233                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2234                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2235
2236                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2237                 long groupId = createGroupId(getGroupIdKey(routerName));
2238
2239                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2240                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2241                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2242             }
2243         }
2244     }
2245
2246     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2247                                          @Nonnull Collection<String> externalIps, @Nullable String vpnName,
2248                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2249             throws ExecutionException, InterruptedException {
2250         //Withdraw the corresponding routes from the BGP.
2251         //Get the network ID using the router ID.
2252         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2253                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2254         if (networkUuid == null) {
2255             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2256             return;
2257         }
2258
2259         if (externalIps.isEmpty()) {
2260             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2261             return;
2262         }
2263
2264         if (vpnName == null) {
2265             //Get the VPN Name using the network ID
2266             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2267             if (vpnName == null) {
2268                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2269                     networkUuid, routerId);
2270                 return;
2271             }
2272         }
2273         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2274
2275         //Remove custom FIB routes
2276         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2277         for (String extIp : externalIps) {
2278             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2279         }
2280     }
2281
2282     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2283                                             final Uuid networkUuid, String extGwMacAddress,
2284                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2285             throws ExecutionException, InterruptedException {
2286         clearBgpRoutes(extIp, vpnName);
2287         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2288                 removeFlowInvTx);
2289     }
2290
2291     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2292                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2293                                              String gwMacAddress, boolean switchOver,
2294                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2295             throws ExecutionException, InterruptedException {
2296         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2297         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2298         if (routerName == null) {
2299             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2300             return;
2301         }
2302         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2303         if (extNwProvType == null) {
2304             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2305             return;
2306         }
2307         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2308          * external network provided type is VxLAN
2309          */
2310         if (extNwProvType == ProviderTypes.VXLAN) {
2311             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2312             );
2313             return;
2314         }
2315         if (tempLabel < 0) {
2316             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2317                     extIp, routerId);
2318             return;
2319         }
2320
2321         final long label = tempLabel;
2322         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2323         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2324                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2325                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2326         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2327
2328         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2329         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2330         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2331             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2332             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2333         }
2334         if (!switchOver) {
2335             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2336                 Futures.transformAsync(future, result -> {
2337                     //Release label
2338                     if (result.isSuccessful()) {
2339                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2340                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2341                             .setVpnName(vpnName).setIpPrefix(externalIp).build();
2342                         return vpnService.removeVpnLabel(labelInput);
2343                     } else {
2344                         String errMsg =
2345                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2346                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2347                         LOG.error(errMsg);
2348                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2349                     }
2350                 }, MoreExecutors.directExecutor());
2351
2352             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2353
2354                 @Override
2355                 public void onFailure(@Nonnull Throwable error) {
2356                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2357                         + "got external ip {}", label, extIp, error);
2358                 }
2359
2360                 @Override
2361                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2362                     if (result.isSuccessful()) {
2363                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2364                             + "from VPN {}", externalIp, vpnName);
2365                     } else {
2366                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2367                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2368                     }
2369                 }
2370             }, MoreExecutors.directExecutor());
2371         } else {
2372             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2373                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2374         }
2375     }
2376
2377     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2378                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2379                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2380             throws ExecutionException, InterruptedException {
2381         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2382         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2383         if (routerName == null) {
2384             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2385             return;
2386         }
2387         //Get the external network provider type from networkId
2388         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2389         if (extNwProvType == null) {
2390             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2391                     networkUuid);
2392             return;
2393         }
2394         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2395          *  external network provided type is VxLAN
2396          */
2397         if (extNwProvType == ProviderTypes.VXLAN) {
2398             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2399             return;
2400         }
2401         //Get IPMaps from the DB for the router ID
2402         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2403         if (dbIpMaps.isEmpty()) {
2404             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2405             return;
2406         }
2407
2408         long tempLabel = NatConstants.INVALID_ID;
2409         for (IpMap dbIpMap : dbIpMaps) {
2410             String dbExternalIp = dbIpMap.getExternalIp();
2411             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2412             //Select the IPMap, whose external IP is the IP for which FIB is installed
2413             if (extIp.equals(dbExternalIp)) {
2414                 tempLabel = dbIpMap.getLabel();
2415                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2416                     tempLabel, dbExternalIp, routerId);
2417                 break;
2418             }
2419         }
2420         if (tempLabel == NatConstants.INVALID_ID) {
2421             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2422                     extIp, routerId);
2423             return;
2424         }
2425
2426         final long label = tempLabel;
2427         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2428         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2429                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2430                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2431         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2432
2433         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2434         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2435         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2436             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2437             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2438         }
2439         if (!switchOver) {
2440             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2441                     Futures.transformAsync(future, result -> {
2442                         //Release label
2443                         if (result.isSuccessful()) {
2444                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2445                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2446                             return vpnService.removeVpnLabel(labelInput);
2447                         } else {
2448                             String errMsg =
2449                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2450                                             + "prefix %s Failed - %s",
2451                                             dpnId, externalIp, result.getErrors());
2452                             LOG.error(errMsg);
2453                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2454                         }
2455                     }, MoreExecutors.directExecutor());
2456
2457             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2458
2459                 @Override
2460                 public void onFailure(@Nonnull Throwable error) {
2461                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2462                 }
2463
2464                 @Override
2465                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2466                     if (result.isSuccessful()) {
2467                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2468                             + "from VPN {}", externalIp, vpnName);
2469                     } else {
2470                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2471                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2472                     }
2473                 }
2474             }, MoreExecutors.directExecutor());
2475         } else {
2476             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2477                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2478         }
2479     }
2480
2481     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2482             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2483             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2484         //Withdraw the corresponding routes from the BGP.
2485         //Get the network ID using the router ID.
2486         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2487                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2488         if (networkUuid == null) {
2489             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2490             return;
2491         }
2492
2493         if (externalIps == null || externalIps.isEmpty()) {
2494             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2495             return;
2496         }
2497
2498         if (vpnName == null) {
2499             //Get the VPN Name using the network ID
2500             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2501             if (vpnName == null) {
2502                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2503                     networkUuid, routerId);
2504                 return;
2505             }
2506         }
2507         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2508
2509         //Remove custom FIB routes
2510         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2511         for (String extIp : externalIps) {
2512             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2513                     writeFlowInvTx);
2514         }
2515     }
2516
2517     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2518         //Inform BGP about the route removal
2519         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2520         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2521         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2522     }
2523
2524     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2525             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2526         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2527         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2528             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2529         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2530     }
2531
2532     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2533             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2534         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2535         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2536         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2537         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2538     }
2539
2540     /**
2541      * router association to vpn.
2542      *
2543      * @param routerName - Name of router
2544      * @param routerId - router id
2545      * @param bgpVpnName BGP VPN name
2546      */
2547     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2548             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2549         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2550         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2551             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2552
2553             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2554
2555             if (bgpVpnId != NatConstants.INVALID_ID) {
2556                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2557                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2558                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2559                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2560                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2561                     getRoutersIdentifier(bgpVpnId), rtrs);
2562
2563                 // Get the allocated Primary NAPT Switch for this router
2564                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2565
2566                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2567                         routerId, bgpVpnId);
2568                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2569
2570                 // Get the group ID
2571                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2572                 createGroupId(getGroupIdKey(routerName));
2573                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2574                         extNwProvType);
2575             }
2576         }
2577     }
2578
2579     /**
2580      * router disassociation from vpn.
2581      *
2582      * @param routerName - Name of router
2583      * @param routerId - router id
2584      * @param bgpVpnName BGP VPN name
2585      */
2586     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2587             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2588         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2589         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2590             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2591             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2592
2593             // Get the allocated Primary NAPT Switch for this router
2594             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2595
2596             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2597             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2598
2599             // Get the group ID
2600             createGroupId(getGroupIdKey(routerName));
2601             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2602             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2603                     writeFlowInvTx, extNwProvType);
2604         }
2605     }
2606
2607     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2608         InstanceIdentifier<Routers> routerInstanceIndentifier =
2609             InstanceIdentifier.builder(ExtRouters.class)
2610                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2611         try {
2612             Optional<Routers> routerData = SingleTransactionDataBroker
2613                     .syncReadOptional(dataBroker,
2614                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2615             return routerData.isPresent() && routerData.get().isEnableSnat();
2616         } catch (ReadFailedException e) {
2617             LOG.error("Failed to read data for router id {}", routerUuid, e);
2618             return false;
2619         }
2620     }
2621
2622     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2623         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2624
2625         long changedVpnId = bgpVpnId;
2626         String idType = "BGP VPN";
2627         if (bgpVpnId == NatConstants.INVALID_ID) {
2628             changedVpnId = routerId;
2629             idType = "router";
2630         }
2631
2632         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2633         if (switches.isEmpty()) {
2634             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2635             return;
2636         }
2637         for (BigInteger dpnId : switches) {
2638             // Update the BGP VPN ID in the SNAT miss entry to group
2639             if (!dpnId.equals(primarySwitchId)) {
2640                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2641                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2642                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2643                 long groupId = createGroupId(getGroupIdKey(routerName));
2644                 if (!isSnatCfgd) {
2645                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2646                 }
2647
2648                 LOG.debug(
2649                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2650                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2651                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2652                 mdsalManager.addFlow(confTx, flowEntity);
2653             } else {
2654                 LOG.debug(
2655                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2656                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2657                 FlowEntity flowEntity =
2658                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2659                 mdsalManager.addFlow(confTx, flowEntity);
2660
2661                 LOG.debug(
2662                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2663                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2664                         idType, changedVpnId, primarySwitchId);
2665                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2666                         changedVpnId, confTx, extNwProvType);
2667
2668                 LOG.debug(
2669                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2670                                 + "which punts the packet to the controller in the Primary switch {}",
2671                         idType, changedVpnId, primarySwitchId);
2672                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2673
2674                 LOG.debug(
2675                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2676                                 + " outgoing packet to FIB Table in the Primary switch {}",
2677                         idType, changedVpnId, primarySwitchId);
2678                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2679
2680                 LOG.debug(
2681                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2682                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2683                                 + " {}", idType, changedVpnId, primarySwitchId);
2684                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2685
2686                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2687                         primarySwitchId);
2688                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2689                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2690                 if (vpnId != NatConstants.INVALID_ID) {
2691                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2692                 }
2693             }
2694         }
2695     }
2696
2697     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2698         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2699         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2700         if (ipPortMapping == null) {
2701             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2702             return;
2703         }
2704         // Get the External Gateway MAC Address
2705         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2706         if (extGwMacAddress != null) {
2707             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2708                     extGwMacAddress, routerId);
2709         } else {
2710             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2711                     routerId);
2712             return;
2713         }
2714         for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2715                 Collections.<IntextIpProtocolType>emptyList())) {
2716             for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2717                     Collections.<IpPortMap>emptyList())) {
2718                 String ipPortInternal = ipPortMap.getIpPortInternal();
2719                 String[] ipPortParts = ipPortInternal.split(":");
2720                 if (ipPortParts.length != 2) {
2721                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2722                     return;
2723                 }
2724                 String internalIp = ipPortParts[0];
2725                 String internalPort = ipPortParts[1];
2726                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2727                         internalIp, internalPort);
2728                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2729                 NAPTEntryEvent.Protocol protocol;
2730                 switch (protocolTypes) {
2731                     case TCP:
2732                         protocol = NAPTEntryEvent.Protocol.TCP;
2733                         break;
2734                     case UDP:
2735                         protocol = NAPTEntryEvent.Protocol.UDP;
2736                         break;
2737                     default:
2738                         protocol = NAPTEntryEvent.Protocol.TCP;
2739                 }
2740                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2741                 SessionAddress externalAddress =
2742                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2743                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2744                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2745                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2746                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2747                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2748             }
2749         }
2750     }
2751
2752     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2753                                                           long changedVpnId) {
2754
2755         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2756             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2757         List<MatchInfo> matches = new ArrayList<>();
2758         matches.add(MatchEthernetType.IPV4);
2759         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2760
2761         List<ActionInfo> actionsInfo = new ArrayList<>();
2762         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2763                 elanManager, idManager, changedVpnId, routerName);
2764         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2765         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2766                 actionsInfo);
2767         actionsInfo.add(new ActionGroup(groupId));
2768         List<InstructionInfo> instructions = new ArrayList<>();
2769         instructions.add(new InstructionApplyActions(actionsInfo));
2770         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2771         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2772             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2773             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2774
2775         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2776         return flowEntity;
2777     }
2778
2779     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2780                                                                         long changedVpnId) {
2781
2782         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2783                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2784         List<MatchInfo> matches = new ArrayList<>();
2785         matches.add(MatchEthernetType.IPV4);
2786         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2787
2788         List<InstructionInfo> instructions = new ArrayList<>();
2789         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2790
2791         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2792         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2793             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2794             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2795
2796         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2797         return flowEntity;
2798     }
2799
2800     // TODO : Replace this with ITM Rpc once its available with full functionality
2801     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2802         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2803
2804         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2805             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2806         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2807                 extNwProvType);
2808         mdsalManager.addFlow(confTx, flowEntity);
2809     }
2810
2811     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2812             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2813         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2814             dpId, routerName, changedVpnId);
2815         List<MatchInfo> matches = new ArrayList<>();
2816         matches.add(MatchEthernetType.IPV4);
2817
2818         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2819         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2820             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2821         }
2822         matches.add(new MatchTunnelId(tunnelId));
2823
2824         List<InstructionInfo> instructions = new ArrayList<>();
2825         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2826             MetaDataUtil.METADATA_MASK_VRFID));
2827         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2828         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2829         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2830         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2831             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2832             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2833         return flowEntity;
2834     }
2835
2836     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2837                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2838         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2839             dpnId, routerId, changedVpnId);
2840         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2841                 NwConstants.IP_PROT_TCP);
2842         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2843         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2844
2845         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2846                 NwConstants.IP_PROT_UDP);
2847         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2848         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2849
2850         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2851         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2852         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2853     }
2854
2855     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2856                                                            long changedVpnId, int protocol) {
2857         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2858             dpId, routerId, changedVpnId);
2859         BigInteger cookie = getCookieOutboundFlow(routerId);
2860         List<MatchInfo> matches = new ArrayList<>();
2861         matches.add(MatchEthernetType.IPV4);
2862         matches.add(new MatchIpProtocol((short)protocol));
2863         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2864
2865         List<InstructionInfo> instructions = new ArrayList<>();
2866         List<ActionInfo> actionsInfos = new ArrayList<>();
2867         actionsInfos.add(new ActionPuntToController());
2868         if (snatPuntTimeout != 0) {
2869             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2870         }
2871         instructions.add(new InstructionApplyActions(actionsInfos));
2872
2873         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2874         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2875             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2876         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2877         return flowEntity;
2878     }
2879
2880     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2881                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2882         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2883             dpnId, segmentId, changedVpnId);
2884         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2885         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2886     }
2887
2888     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2889
2890         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2891             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2892         List<MatchInfo> matches = new ArrayList<>();
2893         matches.add(MatchEthernetType.IPV4);
2894         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2895
2896         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2897         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2898         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2899         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2900         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2901
2902         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2903         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2904             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2905             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2906         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2907         return flowEntity;
2908     }
2909
2910     @Override
2911     protected ExternalRoutersListener getDataTreeChangeListener() {
2912         return ExternalRoutersListener.this;
2913     }
2914
2915     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2916                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2917         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2918                 routerName);
2919         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2920             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2921             if (subnetVpnId != -1) {
2922                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2923                     + "and vpnId {}", dpnId, subnetVpnId);
2924                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2925             }
2926         }
2927     }
2928 }