natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.netvirt.natservice.internal.NatUtil.requireNonNullElse;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import java.math.BigInteger;
  20 import java.net.Inet6Address;
  21 import java.net.InetAddress;
  22 import java.net.UnknownHostException;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Collections;
  27 import java.util.HashMap;
  28 import java.util.HashSet;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Set;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.Future;
  35 import javax.annotation.Nonnull;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  45 import org.opendaylight.genius.infra.Datastore.Configuration;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  48 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  49 import org.opendaylight.genius.infra.TypedWriteTransaction;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.mdsalutil.ActionInfo;
  52 import org.opendaylight.genius.mdsalutil.BucketInfo;
  53 import org.opendaylight.genius.mdsalutil.FlowEntity;
  54 import org.opendaylight.genius.mdsalutil.GroupEntity;
  55 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  56 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  57 import org.opendaylight.genius.mdsalutil.MatchInfo;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  70 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  73 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  74 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  79 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  80 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  81 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  82 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  83 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  84 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  85 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  86 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  87 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  88 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  91 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 150 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 151 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 152 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 153 import org.opendaylight.yangtools.yang.common.RpcResult;
 154 import org.slf4j.Logger;
 155 import org.slf4j.LoggerFactory;
 156
 157
 158 @Singleton
 159 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 160     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 161
 162     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 163     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 164
 165     private final DataBroker dataBroker;
 166     private final ManagedNewTransactionRunner txRunner;
 167     private final IMdsalApiManager mdsalManager;
 168     private final ItmRpcService itmManager;
 169     private final OdlInterfaceRpcService odlInterfaceRpcService;
 170     private final IdManagerService idManager;
 171     private final NaptManager naptManager;
 172     private final NAPTSwitchSelector naptSwitchSelector;
 173     private final IBgpManager bgpManager;
 174     private final VpnRpcService vpnService;
 175     private final FibRpcService fibService;
 176     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 177     private final NaptEventHandler naptEventHandler;
 178     private final NaptPacketInHandler naptPacketInHandler;
 179     private final IFibManager fibManager;
 180     private final IVpnManager vpnManager;
 181     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 182     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 183     private final NatMode natMode;
 184     private final INeutronVpnManager nvpnManager;
 185     private final IElanService elanManager;
 186     private final JobCoordinator coordinator;
 187     private final UpgradeState upgradeState;
 188     private final IInterfaceManager interfaceManager;
 189     private final NatOverVxlanUtil natOverVxlanUtil;
 190     private final int snatPuntTimeout;
 191
 192     @Inject
 193     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 194                                    final ItmRpcService itmManager,
 195                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 196                                    final IdManagerService idManager,
 197                                    final NaptManager naptManager,
 198                                    final NAPTSwitchSelector naptSwitchSelector,
 199                                    final IBgpManager bgpManager,
 200                                    final VpnRpcService vpnService,
 201                                    final FibRpcService fibService,
 202                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 203                                    final NaptEventHandler naptEventHandler,
 204                                    final NaptPacketInHandler naptPacketInHandler,
 205                                    final IFibManager fibManager,
 206                                    final IVpnManager vpnManager,
 207                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 208                                    final INeutronVpnManager nvpnManager,
 209                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 210                                    final NatserviceConfig config,
 211                                    final IElanService elanManager,
 212                                    final JobCoordinator coordinator,
 213                                    final UpgradeState upgradeState,
 214                                    final NatOverVxlanUtil natOverVxlanUtil,
 215                                    final IInterfaceManager interfaceManager) {
 216         super(Routers.class, ExternalRoutersListener.class);
 217         this.dataBroker = dataBroker;
 218         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 219         this.mdsalManager = mdsalManager;
 220         this.itmManager = itmManager;
 221         this.odlInterfaceRpcService = odlInterfaceRpcService;
 222         this.idManager = idManager;
 223         this.naptManager = naptManager;
 224         this.naptSwitchSelector = naptSwitchSelector;
 225         this.bgpManager = bgpManager;
 226         this.vpnService = vpnService;
 227         this.fibService = fibService;
 228         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 229         this.naptEventHandler = naptEventHandler;
 230         this.naptPacketInHandler = naptPacketInHandler;
 231         this.fibManager = fibManager;
 232         this.vpnManager = vpnManager;
 233         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 234         this.nvpnManager = nvpnManager;
 235         this.elanManager = elanManager;
 236         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 237         this.coordinator = coordinator;
 238         this.upgradeState = upgradeState;
 239         this.interfaceManager = interfaceManager;
 240         this.natOverVxlanUtil = natOverVxlanUtil;
 241         if (config != null) {
 242             this.natMode = config.getNatMode();
 243             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 244         } else {
 245             this.natMode = NatMode.Controller;
 246             this.snatPuntTimeout = 0;
 247         }
 248     }
 249
 250     @Override
 251     @PostConstruct
 252     public void init() {
 253         LOG.info("{} init", getClass().getSimpleName());
 254         // This class handles ExternalRouters for Controller SNAT mode.
 255         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 256         if (natMode == NatMode.Controller) {
 257             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 258             NatUtil.createGroupIdPool(idManager);
 259         }
 260     }
 261
 262     @Override
 263     protected InstanceIdentifier<Routers> getWildCardPath() {
 264         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 265     }
 266
 267     @Override
 268     // TODO Clean up the exception handling
 269     @SuppressWarnings("checkstyle:IllegalCatch")
 270     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 271         // Populate the router-id-name container
 272         String routerName = routers.getRouterName();
 273         LOG.info("add : external router event for {}", routerName);
 274         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 275         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 276         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 277         try {
 278             if (routers.isEnableSnat()) {
 279                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 280                     () -> Collections.singletonList(
 281                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 282                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 283                             long bgpVpnId = NatConstants.INVALID_ID;
 284                             if (bgpVpnUuid != null) {
 285                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 286                             }
 287                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 288                             // Allocate Primary Napt Switch for this router
 289                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 290                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 291                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 292                             }
 293                         }
 294                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 295             } else {
 296                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 297             }
 298         } catch (Exception ex) {
 299             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 300                     routerName, ex);
 301         }
 302     }
 303
 304     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 305                                  TypedWriteTransaction<Configuration> confTx) {
 306         String routerName = routers.getRouterName();
 307         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 308
 309         naptManager.initialiseExternalCounter(routers, routerId);
 310         subnetRegisterMapping(routers, routerId);
 311
 312         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 313             primarySwitchId, routerName);
 314
 315         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 316                 routers.getNetworkId());
 317         if (extNwProvType == null) {
 318             LOG.error("handleEnableSnat : External Network Provider Type missing");
 319             return;
 320         }
 321
 322         if (bgpVpnId != NatConstants.INVALID_ID) {
 323             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 324                     extNwProvType);
 325         } else {
 326             // write metadata and punt
 327             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 328             // Now install entries in SNAT tables to point to Primary for each router
 329             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 330             for (BigInteger dpnId : switches) {
 331                 // Handle switches and NAPT switches separately
 332                 if (!dpnId.equals(primarySwitchId)) {
 333                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 334                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 335                 } else {
 336                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 337                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 338                 }
 339             }
 340         }
 341
 342         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 343         if (externalIps.isEmpty()) {
 344             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 345             return;
 346         } else {
 347             for (String externalIpAddrPrefix : externalIps) {
 348                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 349                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 350                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 351                 );
 352             }
 353         }
 354         LOG.debug("handleEnableSnat : Exit");
 355     }
 356
 357     private BigInteger getPrimaryNaptSwitch(String routerName) {
 358         // Allocate Primary Napt Switch for this router
 359         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 360         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 361             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 362                 primarySwitchId, routerName);
 363             return primarySwitchId;
 364         }
 365         // Allocated an id from VNI pool for the Router.
 366         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 367         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 368         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 369
 370         return primarySwitchId;
 371     }
 372
 373     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 374                                                      TypedWriteTransaction<Configuration> confTx) {
 375         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 376         if (extVpnId == NatConstants.INVALID_ID) {
 377             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 378             extVpnId = routerId;
 379         }
 380         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 381         if (externalIps.isEmpty()) {
 382             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 383                 dpnId, extVpnId);
 384             return;
 385         }
 386         for (String ip : externalIps) {
 387             Uuid subnetId = getSubnetIdForFixedIp(ip);
 388             if (subnetId != null) {
 389                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 390                 if (subnetVpnId != NatConstants.INVALID_ID) {
 391                     extVpnId = subnetVpnId;
 392                 }
 393                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 394                     dpnId, extVpnId, subnetId);
 395                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 396                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 397             }
 398         }
 399     }
 400
 401     @Nullable
 402     private Uuid getSubnetIdForFixedIp(String ip) {
 403         if (ip != null) {
 404             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 405             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 406             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 407         }
 408         LOG.error("getSubnetIdForFixedIp : ip is null");
 409         return null;
 410     }
 411
 412     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 413         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 414         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 415         int counter = 0;
 416         int extIpCounter = externalIps.size();
 417         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 418                 counter, extIpCounter);
 419         for (Uuid subnet : requireNonNullElse(routerEntry.getSubnetIds(), Collections.<Uuid>emptyList())) {
 420             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 421             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 422                 .builder(Subnetmaps.class)
 423                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 424                 .build();
 425             Optional<Subnetmap> sn;
 426             try {
 427                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 428                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 429             } catch (ReadFailedException e) {
 430                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 431                 sn = Optional.absent();
 432             }
 433             if (sn.isPresent()) {
 434                 // subnets
 435                 Subnetmap subnetmapEntry = sn.get();
 436                 String subnetString = subnetmapEntry.getSubnetIp();
 437                 String[] subnetSplit = subnetString.split("/");
 438                 String subnetIp = subnetSplit[0];
 439                 try {
 440                     InetAddress address = InetAddress.getByName(subnetIp);
 441                     if (address instanceof Inet6Address) {
 442                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 443                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 444                         continue;
 445                     }
 446                 } catch (UnknownHostException e) {
 447                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 448                     return;
 449                 }
 450                 String subnetPrefix = "0";
 451                 if (subnetSplit.length == 2) {
 452                     subnetPrefix = subnetSplit[1];
 453                 }
 454                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 455                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 456                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 457                 //externalIps
 458                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 459                         counter, extIpCounter);
 460                 if (extIpCounter != 0) {
 461                     if (counter < extIpCounter) {
 462                         String[] ipSplit = externalIps.get(counter).split("/");
 463                         String externalIp = ipSplit[0];
 464                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 465                         if (ipSplit.length == 2) {
 466                             extPrefix = ipSplit[1];
 467                         }
 468                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 469                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 470                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 471                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 472                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 473                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 474                     } else {
 475                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 476                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 477                         String[] ipSplit = externalIps.get(counter).split("/");
 478                         String externalIp = ipSplit[0];
 479                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 480                         if (ipSplit.length == 2) {
 481                             extPrefix = ipSplit[1];
 482                         }
 483                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 484                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 485                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 486                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 487                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 488                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 489                             externalIp, extPrefix);
 490                     }
 491                 }
 492                 counter++;
 493                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 494             } else {
 495                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 496             }
 497         }
 498     }
 499
 500     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 501             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 502             throws ExecutionException, InterruptedException {
 503         //Check if BGP VPN exists. If exists then invoke the new method.
 504         if (bgpVpnId != NatConstants.INVALID_ID) {
 505             if (bgpVpnUuid != null) {
 506                 String bgpVpnName = bgpVpnUuid.getValue();
 507                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 508                     bgpVpnId, bgpVpnName);
 509                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 510                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 511                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 512             }
 513             if (create) {
 514                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 515             } else {
 516                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 517             }
 518             return;
 519         }
 520
 521         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 522         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 523     }
 524
 525     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 526             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 527         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 528         if (switches.isEmpty()) {
 529             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 530             return;
 531         }
 532         if (routerId == NatConstants.INVALID_ID) {
 533             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 534                     + "default NAT route in FIB", routerName);
 535             return;
 536         }
 537         for (BigInteger dpnId : switches) {
 538             if (create) {
 539                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 540                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 541                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 542             } else {
 543                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 544                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 545                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 546             }
 547         }
 548     }
 549
 550     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 551         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 552         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 553         if (dpnIds.isEmpty()) {
 554             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 555                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 556             return;
 557         }
 558         for (BigInteger dpnId : dpnIds) {
 559             if (bgpVpnId != NatConstants.INVALID_ID) {
 560                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 561                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 562                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 563             } else {
 564                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 565                     + "in dpn {} for the internal vpn", routerId, dpnId);
 566                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 567             }
 568         }
 569     }
 570
 571     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 572             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 573             throws ExecutionException, InterruptedException {
 574         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 575         if (dpnIds.isEmpty()) {
 576             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 577                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 578             return;
 579         }
 580         for (BigInteger dpnId : dpnIds) {
 581             if (bgpVpnId != NatConstants.INVALID_ID) {
 582                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 583                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 584                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 585             } else {
 586                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 587                     + "in dpn {} for the internal vpn", routerId, dpnId);
 588                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 589             }
 590         }
 591     }
 592
 593     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 594         TypedWriteTransaction<Configuration> confTx) {
 595         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 596         if (routerId != NatConstants.INVALID_ID) {
 597             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 598                     primarySwitchId, routerId);
 599             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 600         } else {
 601             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 602                 + "createAndInstallMissEntry", routerName);
 603         }
 604     }
 605
 606     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 607         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 608                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 609     }
 610
 611     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 612         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 613                 .FLOWID_SEPARATOR + vpnId;
 614     }
 615
 616     public BigInteger getCookieOutboundFlow(long routerId) {
 617         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 618             BigInteger.valueOf(routerId));
 619     }
 620
 621     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 622         long l4SrcPortField;
 623         long l4DstPortField;
 624         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 625
 626         if (protocol == NwConstants.IP_PROT_TCP) {
 627             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 628             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 629         } else {
 630             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 631             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 632         }
 633         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 634                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 635                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 636                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 637                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 638                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 639                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 640                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 641                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 642                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 643                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 644                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 645                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 646                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 647                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 648
 649         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 650                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 651     }
 652
 653     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 654         List<MatchInfo> matches = new ArrayList<>();
 655         matches.add(MatchEthernetType.IPV4);
 656         matches.add(MatchIpProtocol.ICMP);
 657         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 658
 659         List<ActionInfo> actionInfos = new ArrayList<>();
 660         actionInfos.add(new ActionDrop());
 661
 662         List<InstructionInfo> instructions = new ArrayList<>();
 663         instructions.add(new InstructionApplyActions(actionInfos));
 664
 665         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 666                 NwConstants.IP_PROT_ICMP);
 667         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 668                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 669                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 670         return flow;
 671     }
 672
 673     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 674         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 675         BigInteger cookie = getCookieOutboundFlow(routerId);
 676         List<MatchInfo> matches = new ArrayList<>();
 677         matches.add(MatchEthernetType.IPV4);
 678         matches.add(new MatchIpProtocol((short)protocol));
 679         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 680
 681         List<InstructionInfo> instructions = new ArrayList<>();
 682         List<ActionInfo> actionsInfos = new ArrayList<>();
 683         actionsInfos.add(new ActionPuntToController());
 684         if (snatPuntTimeout != 0) {
 685             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 686         }
 687         instructions.add(new InstructionApplyActions(actionsInfos));
 688
 689         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 690         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 691             5, flowRef, 0, 0,
 692             cookie, matches, instructions);
 693         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 694         return flowEntity;
 695     }
 696
 697     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 698         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 699         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 700         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 701         mdsalManager.addFlow(confTx, tcpflowEntity);
 702
 703         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 704         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 705         mdsalManager.addFlow(confTx, udpflowEntity);
 706
 707         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 708         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 709         mdsalManager.addFlow(confTx, icmpDropFlow);
 710     }
 711
 712     @Nullable
 713     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 714         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 715         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 716         try {
 717             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 718                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 719                     .setSourceDpid(srcDpId)
 720                     .setDestinationDpid(dstDpId)
 721                     .setTunnelType(tunType)
 722                     .build());
 723             rpcResult = result.get();
 724             if (!rpcResult.isSuccessful()) {
 725                 tunType = TunnelTypeGre.class;
 726                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 727                     .setSourceDpid(srcDpId)
 728                     .setDestinationDpid(dstDpId)
 729                     .setTunnelType(tunType)
 730                     .build());
 731                 rpcResult = result.get();
 732                 if (!rpcResult.isSuccessful()) {
 733                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 734                             rpcResult.getErrors());
 735                 } else {
 736                     return rpcResult.getResult().getInterfaceName();
 737                 }
 738                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 739                         rpcResult.getErrors());
 740             } else {
 741                 return rpcResult.getResult().getInterfaceName();
 742             }
 743         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 744             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 745                     + "between {} and {}", srcDpId, dstDpId, e);
 746         }
 747
 748         return null;
 749     }
 750
 751     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 752         TypedWriteTransaction<Configuration> confTx) {
 753
 754         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 755         // Install miss entry pointing to group
 756         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 757         mdsalManager.addFlow(confTx, flowEntity);
 758     }
 759
 760     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 761             String routerName, long routerId) {
 762         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 763             dpnId, bucketInfo.get(0));
 764         // Install the select group
 765         long groupId = createGroupId(getGroupIdKey(routerName));
 766         GroupEntity groupEntity =
 767             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 768         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 769         mdsalManager.syncInstallGroup(groupEntity);
 770         // Install miss entry pointing to group
 771         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 772         if (flowEntity == null) {
 773             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 774                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 775                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 776             return;
 777         }
 778         mdsalManager.installFlow(flowEntity);
 779     }
 780
 781     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 782         long groupId = createGroupId(getGroupIdKey(routerName));
 783         GroupEntity groupEntity =
 784             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 785         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 786         mdsalManager.syncInstallGroup(groupEntity);
 787         return groupId;
 788     }
 789
 790     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 791         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 792                 dpId, routerName, groupId);
 793         List<MatchInfo> matches = new ArrayList<>();
 794         matches.add(MatchEthernetType.IPV4);
 795         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 796
 797         List<ActionInfo> actionsInfo = new ArrayList<>();
 798         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 799                 routerId, routerName);
 800         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 801         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 802         actionsInfo.add(new ActionGroup(groupId));
 803         List<InstructionInfo> instructions = new ArrayList<>();
 804         instructions.add(new InstructionApplyActions(actionsInfo));
 805         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 806         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 807                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 808                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 809
 810         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 811         return flowEntity;
 812     }
 813
 814     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 815
 816         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 817             routerName);
 818         List<MatchInfo> matches = new ArrayList<>();
 819         matches.add(MatchEthernetType.IPV4);
 820         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 821
 822         List<InstructionInfo> instructions = new ArrayList<>();
 823         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 824
 825         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 826         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 827             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 828             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 829
 830         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 831         return flowEntity;
 832     }
 833
 834     // TODO : Replace this with ITM Rpc once its available with full functionality
 835     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 836         TypedWriteTransaction<Configuration> confTx) {
 837
 838         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 839             dpnId, routerName);
 840         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 841         if (flowEntity == null) {
 842             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 843                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 844                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 845                     dpnId, routerName);
 846             return;
 847         }
 848         mdsalManager.addFlow(confTx, flowEntity);
 849
 850     }
 851
 852     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 853         List<MatchInfo> matches = new ArrayList<>();
 854         matches.add(MatchEthernetType.IPV4);
 855         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 856                 idManager, routerId, routerName);
 857         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 858         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 859         List<InstructionInfo> instructions = new ArrayList<>();
 860         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 861                 MetaDataUtil.METADATA_MASK_VRFID));
 862         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 863         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 864                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 865                 instructions);
 866         return flowEntity;
 867     }
 868
 869     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 870         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 871                 .FLOWID_SEPARATOR + routerID;
 872     }
 873
 874     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 875         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 876                 .FLOWID_SEPARATOR + routerID;
 877     }
 878
 879     private String getGroupIdKey(String routerName) {
 880         return "snatmiss." + routerName;
 881     }
 882
 883     protected long createGroupId(String groupIdKey) {
 884         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 885             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 886             .build();
 887         try {
 888             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 889             RpcResult<AllocateIdOutput> rpcResult = result.get();
 890             return rpcResult.getResult().getIdValue();
 891         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 892             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 893         }
 894         return 0;
 895     }
 896
 897     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 898         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 899         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 900         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 901         List<BucketInfo> listBucketInfo = new ArrayList<>();
 902
 903         if (ifNamePrimary != null) {
 904             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 905             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 906                     interfaceManager, ifNamePrimary, routerId, true);
 907             if (listActionInfoPrimary.isEmpty()) {
 908                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 909                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 910                         ifNamePrimary);
 911             }
 912         } else {
 913             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 914                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 915         }
 916         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 917
 918         listBucketInfo.add(0, bucketPrimary);
 919         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 920     }
 921
 922     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 923                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 924         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 925         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 926         List<BucketInfo> listBucketInfo = new ArrayList<>();
 927
 928         if (ifNamePrimary != null) {
 929             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 930                     ifNamePrimary);
 931             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 932                     interfaceManager, ifNamePrimary, routerId, true);
 933             if (listActionInfoPrimary.isEmpty()) {
 934                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 935                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 936                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 937             }
 938         } else {
 939             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 940                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 941         }
 942         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 943
 944         listBucketInfo.add(0, bucketPrimary);
 945         return listBucketInfo;
 946     }
 947
 948     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 949         TypedWriteTransaction<Configuration> confTx) {
 950
 951        /*
 952         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 953         */
 954
 955         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 956
 957 /*
 958         List<BucketInfo> listBucketInfo = new ArrayList<>();
 959         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 960         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 961         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 962         listBucketInfo.add(0, bucketPrimary);
 963 */
 964
 965         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 966         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 967         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 968         installNaptPfibEntry(dpnId, routerId, confTx);
 969         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 970         if (networkId != null) {
 971             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 972             if (vpnUuid != null) {
 973                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 974                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 975                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 976                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 977                     if (vpnId != NatConstants.INVALID_ID) {
 978                         installNaptPfibEntry(dpnId, vpnId, null);
 979                     }
 980                     return Collections.emptyList();
 981                 });
 982             } else {
 983                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 984             }
 985         } else {
 986             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 987         }
 988     }
 989
 990     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 991         List<BucketInfo> listBucketInfo = new ArrayList<>();
 992         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 993         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 994         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 995         listBucketInfo.add(0, bucketPrimary);
 996         return listBucketInfo;
 997     }
 998
 999     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
1000             @Nullable TypedWriteTransaction<Configuration> confTx) {
1001         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1002         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1003         if (confTx != null) {
1004             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1005         } else {
1006             mdsalManager.installFlow(naptPfibFlowEntity);
1007         }
1008     }
1009
1010     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1011
1012         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1013         List<MatchInfo> matches = new ArrayList<>();
1014         matches.add(MatchEthernetType.IPV4);
1015         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1016
1017         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1018         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1019         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1020         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1021         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1022
1023         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1024         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1025             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1026             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1027         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1028         return flowEntity;
1029     }
1030
1031     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1032         long routerId, String routerName, String externalIp) {
1033         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1034             dpnId, routerId, externalIp);
1035         Uuid networkId = router.getNetworkId();
1036         if (networkId == null) {
1037             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1038             return;
1039         }
1040         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1041         if (vpnName == null) {
1042             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1043                     + "configuration {} in router {}", networkId, externalIp, routerId);
1044             return;
1045         }
1046         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1047             externalIp, networkId, router, confTx);
1048         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1049             dpnId, routerId, externalIp);
1050     }
1051
1052     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1053                                                 final long routerId, final String routerName, final String externalIp,
1054                                                 final Uuid extNetworkId, @Nullable final Routers router,
1055                                                 final TypedWriteTransaction<Configuration> confTx) {
1056         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1057                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1058         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1059         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1060         if (rd == null || rd.isEmpty()) {
1061             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1062             return;
1063         }
1064         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1065         if (extNwProvType == null) {
1066             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1067             return;
1068         }
1069         if (extNwProvType == ProviderTypes.VXLAN) {
1070             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1071                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1072             return;
1073         }
1074         //Generate VPN label for the external IP
1075         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1076             .setIpPrefix(externalIp).build();
1077         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1078
1079         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1080         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1081             if (result.isSuccessful()) {
1082                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1083                 GenerateVpnLabelOutput output = result.getResult();
1084                 final long label = output.getLabel();
1085
1086                 int externalIpInDsFlag = 0;
1087                 //Get IPMaps from the DB for the router ID
1088                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1089                 for (IpMap dbIpMap : dbIpMaps) {
1090                     String dbExternalIp = dbIpMap.getExternalIp();
1091                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1092                     if (dbExternalIp.contains(externalIp)) {
1093                         String dbInternalIp = dbIpMap.getInternalIp();
1094                         IpMapKey dbIpMapKey = dbIpMap.key();
1095                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1096                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1097                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1098                             .setExternalIp(dbExternalIp).setLabel(label).build();
1099                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1100                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1101                         externalIpInDsFlag++;
1102                     }
1103                 }
1104                 if (externalIpInDsFlag <= 0) {
1105                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1106                             + "Failed to update label {} for routerId {} in DS",
1107                             externalIp, label, routerId);
1108                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1109                         + " found in DS for router %s", label, externalIp, routerId);
1110                     return Futures.immediateFailedFuture(new Exception(errMsg));
1111                 }
1112                 //Inform BGP
1113                 long l3vni = 0;
1114                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1115                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1116                 }
1117                 Routers extRouter = router != null ? router :
1118                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1119                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1120                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1121                     RouteOrigin.STATIC, dpnId);
1122
1123                 //Install custom FIB routes
1124                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1125                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1126                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1127                         extNwProvType);
1128                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1129
1130                 //Install custom FIB routes - FIB table.
1131                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1132                         routerName, externalIp);
1133                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1134                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1135                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1136                 }
1137                 String externalVpn = vpnName;
1138                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1139                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1140                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1141                             externalSubnetId);
1142                     if (externalSubnet.isPresent()) {
1143                         externalVpn =  externalSubnetId.getValue();
1144                     }
1145                 }
1146                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1147                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1148                     .setVpnName(externalVpn)
1149                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1150                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1151                     .setInstruction(fibTableCustomInstructions).build();
1152                 return fibService.createFibEntry(input);
1153             } else {
1154                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1155                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1156                     externalIp, vpnName, result.getErrors());
1157                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1158             }
1159         }, MoreExecutors.directExecutor());
1160
1161         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1162
1163             @Override
1164             public void onFailure(@Nonnull Throwable error) {
1165                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1166             }
1167
1168             @Override
1169             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1170                 if (result.isSuccessful()) {
1171                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1172                             externalIp);
1173                 } else {
1174                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1175                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1176                 }
1177             }
1178         }, MoreExecutors.directExecutor());
1179     }
1180
1181     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1182             String externalIp) {
1183         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1184         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1185         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1186                 externalIp, router);
1187         int instructionIndex = 0;
1188         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1189             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1190             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1191                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1192             instructionIndex++;
1193         }
1194
1195         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1196         return fibTableCustomInstructions;
1197     }
1198
1199     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1200         TypedWriteTransaction<Configuration> confTx) {
1201         List<MatchInfo> matches = new ArrayList<>();
1202         matches.add(MatchEthernetType.MPLS_UNICAST);
1203         matches.add(new MatchMplsLabel(serviceId));
1204
1205         List<Instruction> instructions = new ArrayList<>();
1206         List<ActionInfo> actionsInfos = new ArrayList<>();
1207         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1208         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1209         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1210         instructions.add(writeInstruction);
1211         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1212
1213         // Install the flow entry in L3_LFIB_TABLE
1214         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1215
1216         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1217             10, flowRef, 0, 0,
1218             COOKIE_VM_LFIB_TABLE, matches, instructions);
1219
1220         mdsalManager.addFlow(confTx, dpId, flowEntity);
1221
1222         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1223     }
1224
1225     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1226         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1227         ProviderTypes extNwProvType) {
1228
1229         List<MatchInfo> mkMatches = new ArrayList<>();
1230
1231         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1232             dpnId, serviceId, customInstructions);
1233
1234         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1235             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1236         } else {
1237             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1238         }
1239
1240         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1241             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1242             String.format("%s:%d", "TST Flow Entry ", serviceId),
1243             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1244
1245         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1246     }
1247
1248     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1249         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1250             new RouterIdsKey(routerId)).build();
1251     }
1252
1253     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1254         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1255                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1256     }
1257
1258     @Override
1259     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1260         String routerName = original.getRouterName();
1261         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1262         if (routerId == NatConstants.INVALID_ID) {
1263             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1264             return;
1265         }
1266         // Check if its update on SNAT flag
1267         boolean originalSNATEnabled = original.isEnableSnat();
1268         boolean updatedSNATEnabled = update.isEnableSnat();
1269         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1270             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1271         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1272          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1273          */
1274         long bgpVpnId = NatConstants.INVALID_ID;
1275         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1276         if (bgpVpnUuid != null) {
1277             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1278         }
1279         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1280         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1281             // Router has no interface attached
1282             return;
1283         }
1284         final long finalBgpVpnId = bgpVpnId;
1285         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1286             List<ListenableFuture<Void>> futures = new ArrayList<>();
1287             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1288                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1289                     Uuid networkId = original.getNetworkId();
1290                     if (originalSNATEnabled != updatedSNATEnabled) {
1291                         if (originalSNATEnabled) {
1292                             //SNAT disabled for the router
1293                             Uuid networkUuid = original.getNetworkId();
1294                             LOG.info("update : SNAT disabled for Router {}", routerName);
1295                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1296                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1297                                     removeFlowInvTx);
1298                         } else {
1299                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1300                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1301                                     true, writeFlowInvTx);
1302                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1303                         }
1304                     }
1305                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1306                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1307                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1308                     }
1309
1310                     //Check if the Update is on External IPs
1311                     LOG.debug("update : Checking if this is update on External IPs");
1312                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1313                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1314
1315                     //Check if the External IPs are added during the update.
1316                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1317                     addedExternalIps.removeAll(originalExternalIps);
1318                     if (addedExternalIps.size() != 0) {
1319                         LOG.debug("update : Start processing of the External IPs addition during the update "
1320                                 + "operation");
1321                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1322                                 update.getExtGwMacAddress(), dpnId,
1323                                 update.getNetworkId());
1324
1325                         for (String addedExternalIp : addedExternalIps) {
1326                 /*
1327                     1) Do nothing in the IntExtIp model.
1328                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1329                  */
1330                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1331                             String externalIp = externalIpParts[0];
1332                             String externalIpPrefix = externalIpParts[1];
1333                             String externalpStr = externalIp + "/" + externalIpPrefix;
1334                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1335                                             + "router ID {} in the ExternalIpsCounter model.",
1336                                     externalpStr, routerId);
1337                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1338                         }
1339                         LOG.debug(
1340                                 "update : End processing of the External IPs addition during the update operation");
1341                     }
1342
1343                     //Check if the External IPs are removed during the update.
1344                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1345                     removedExternalIps.removeAll(updatedExternalIps);
1346                     if (removedExternalIps.size() > 0) {
1347                         LOG.debug("update : Start processing of the External IPs removal during the update "
1348                                 + "operation");
1349                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1350                                 removedExternalIps, original.getExtGwMacAddress(),
1351                                 dpnId, networkId);
1352
1353                         for (String removedExternalIp : removedExternalIps) {
1354                 /*
1355                     1) Remove the mappings in the IntExt IP model which has external IP.
1356                     2) Remove the external IP in the ExternalCounter model.
1357                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1358                        least loaded external IP.
1359                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1360                     4) Increase the count of the allocated external IP by one.
1361                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1362                      i.e. the route for the external IP is absent.
1363                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1364                      the removed external IPs and also from the model.
1365                     7) Advertise to the BGP for removing the route for the removed external IPs.
1366                  */
1367
1368                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1369                             String externalIp = externalIpParts[0];
1370                             String externalIpPrefix = externalIpParts[1];
1371                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1372
1373                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1374                                     + "entries for removed external IP {}", externalIpAddrStr);
1375                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1376                             String vpnName = "";
1377                             if (vpnUuId != null) {
1378                                 vpnName = vpnUuId.getValue();
1379                             }
1380                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1381                                     update.getExtGwMacAddress(), removeFlowInvTx);
1382
1383                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1384                             //Get the internal IPs which are associated to the removed external IPs
1385                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1386                             List<String> removedInternalIps = new ArrayList<>();
1387                             for (IpMap ipMap : ipMaps) {
1388                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1389                                     removedInternalIps.add(ipMap.getInternalIp());
1390                                 }
1391                             }
1392
1393                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1394                             for (String removedInternalIp : removedInternalIps) {
1395                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1396                                                 + "router ID {} from the IntExtIP model",
1397                                         removedInternalIp, routerId);
1398                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1399                             }
1400
1401                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1402                                             + "router ID {} from the ExternalIpsCounter model.",
1403                                     externalIpAddrStr, routerId);
1404                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1405
1406                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1407                                     + "whose external IPs were removed.");
1408                             for (String removedInternalIp : removedInternalIps) {
1409                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1410                                         removedInternalIp, writeFlowInvTx);
1411                             }
1412                             LOG.debug("update : Remove the NAPT translation entries from "
1413                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1414                             //Get the internalIP and internal Port which were associated to the removed external IP.
1415                             List<Integer> externalPorts = new ArrayList<>();
1416                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1417                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1418                                     .builder(IntextIpPortMap.class)
1419                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1420                             Optional<IpPortMapping> ipPortMapping;
1421                             try {
1422                                 ipPortMapping = SingleTransactionDataBroker
1423                                             .syncReadOptional(dataBroker,
1424                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1425                             } catch (ReadFailedException e) {
1426                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1427                                 ipPortMapping = Optional.absent();
1428                             }
1429
1430                             if (ipPortMapping.isPresent()) {
1431                                 for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(
1432                                         ipPortMapping.get().getIntextIpProtocolType(),
1433                                         Collections.<IntextIpProtocolType>emptyList())) {
1434                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1435                                     for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
1436                                             Collections.<IpPortMap>emptyList())) {
1437                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1438                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1439                                             externalPorts.add(ipPortExternal.getPortNum());
1440                                             List<String> removedInternalIpPorts =
1441                                                     protoTypesIntIpPortsMap.get(protoType);
1442                                             if (removedInternalIpPorts != null) {
1443                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1444                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1445                                             } else {
1446                                                 removedInternalIpPorts = new ArrayList<>();
1447                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1448                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1449                                             }
1450                                         }
1451                                     }
1452                                 }
1453                             }
1454
1455                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1456                             // the removed external IP.
1457                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1458                                     protoTypesIntIpPortsMap.entrySet();
1459                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1460                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1461                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1462                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1463                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1464                                     // Remove the IP port map from the intext-ip-port-map model,
1465                                     // which were containing the removed external IP
1466                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1467                                             protocolType);
1468                                     //Remove the IP port incomint packer map.
1469                                     naptPacketInHandler.removeIncomingPacketMap(
1470                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1471                                     String[] removedInternalIpPortParts = removedInternalIpPort
1472                                             .split(NatConstants.COLON_SEPARATOR);
1473                                     if (removedInternalIpPortParts.length == 2) {
1474                                         String removedInternalIp = removedInternalIpPortParts[0];
1475                                         String removedInternalPort = removedInternalIpPortParts[1];
1476                                         List<String> removedInternalPortsList =
1477                                                 internalIpPortMap.get(removedInternalPort);
1478                                         if (removedInternalPortsList != null) {
1479                                             removedInternalPortsList.add(removedInternalPort);
1480                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1481                                         } else {
1482                                             removedInternalPortsList = new ArrayList<>();
1483                                             removedInternalPortsList.add(removedInternalPort);
1484                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1485                                         }
1486                                     }
1487                                 }
1488                             }
1489
1490                             // Delete the entry from SnatIntIpPortMap DS
1491                             Set<String> internalIps = internalIpPortMap.keySet();
1492                             for (String internalIp : internalIps) {
1493                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1494                                         + "model SnatIntIpPortMap", internalIp);
1495                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1496                             }
1497
1498                             naptManager.removeNaptPortPool(externalIp);
1499
1500                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1501                                     + "the removed external IP {}", externalIp);
1502                             for (Integer externalPort : externalPorts) {
1503                                 //Remove the NAPT translation entries from Inbound NAPT table
1504                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1505                                         routerId, externalIp, externalPort);
1506                             }
1507
1508                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1509                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1510                                 String internalIp = internalIpPort.getKey();
1511                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1512                                         + "for the removed internal IP {}", internalIp);
1513                                 List<String> internalPorts = internalIpPort.getValue();
1514                                 for (String internalPort : internalPorts) {
1515                                     //Remove the NAPT translation entries from Outbound NAPT table
1516                                     naptPacketInHandler.removeIncomingPacketMap(
1517                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1518                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1519                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1520                                             routerId, internalIp, Integer.parseInt(internalPort));
1521                                 }
1522                             }
1523                         }
1524                         LOG.debug(
1525                                 "update : End processing of the External IPs removal during the update operation");
1526                     }
1527
1528                     //Check if its Update on subnets
1529                     LOG.debug("update : Checking if this is update on subnets");
1530                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1531                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1532                     Set<Uuid> addedSubnetIds =
1533                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1534                     if (originalSubnetIds != null) {
1535                         addedSubnetIds.removeAll(originalSubnetIds);
1536                     }
1537
1538                     //Check if the Subnet IDs are added during the update.
1539                     if (addedSubnetIds.size() != 0) {
1540                         LOG.debug(
1541                                 "update : Start processing of the Subnet IDs addition during the update operation");
1542                         for (Uuid addedSubnetId : addedSubnetIds) {
1543                 /*
1544                     1) Select the least loaded external IP for the subnet and store the mapping of the
1545                     subnet IP and the external IP in the IntExtIp model.
1546                     2) Increase the count of the selected external IP by one.
1547                     3) Advertise to the BGP if external IP is allocated for the first time for the
1548                     router i.e. the route for the external IP is absent.
1549                  */
1550                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1551                             if (subnetIp != null) {
1552                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1553                                         writeFlowInvTx);
1554                             }
1555                         }
1556                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1557                     }
1558
1559                     //Check if the Subnet IDs are removed during the update.
1560                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1561                     removedSubnetIds.removeAll(updatedSubnetIds);
1562                     if (removedSubnetIds.size() != 0) {
1563                         LOG.debug(
1564                                 "update : Start processing of the Subnet IDs removal during the update operation");
1565                         for (Uuid removedSubnetId : removedSubnetIds) {
1566                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1567                             if (subnetAddr != null) {
1568                     /*
1569                         1) Remove the subnet IP and the external IP in the IntExtIp map
1570                         2) Decrease the count of the coresponding external IP by one.
1571                         3) Advertise to the BGP for removing the routes of the corresponding external
1572                         IP if its not allocated to any other internal IP.
1573                     */
1574
1575                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1576                                         subnetAddr[0] + "/" + subnetAddr[1]);
1577                                 if (externalIp == null) {
1578                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1579                                             routerId, subnetAddr[0]);
1580                                     return;
1581                                 }
1582
1583                                 naptManager.updateCounter(routerId, externalIp, false);
1584                                 // Traverse entire model of external-ip counter whether external ip is not
1585                                 // used by any other internal ip in any router
1586                                 if (!isExternalIpAllocated(externalIp)) {
1587                                     LOG.debug("update : external ip is not allocated to any other "
1588                                             + "internal IP so proceeding to remove routes");
1589                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1590                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1591                                             removeFlowInvTx);
1592                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1593                                                     + "router {} with networkId {} and externalIp {}",
1594                                             dpnId, routerId, networkId, externalIp);
1595                                 }
1596
1597                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1598                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1599                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1600                             }
1601                         }
1602                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1603                     }
1604                 }));
1605             }));
1606             return futures;
1607         }, NatConstants.NAT_DJC_MAX_RETRIES);
1608     }
1609
1610     private boolean isExternalIpAllocated(String externalIp) {
1611         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1612         Optional<ExternalIpsCounter> externalCountersData;
1613         try {
1614             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1615                         LogicalDatastoreType.OPERATIONAL, id);
1616         } catch (ReadFailedException e) {
1617             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1618             externalCountersData = Optional.absent();
1619         }
1620         if (externalCountersData.isPresent()) {
1621             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1622             for (ExternalCounters ext : requireNonNullElse(externalIpsCounters.getExternalCounters(),
1623                     Collections.<ExternalCounters>emptyList())) {
1624                 for (ExternalIpCounter externalIpCount : requireNonNullElse(ext.getExternalIpCounter(),
1625                         Collections.<ExternalIpCounter>emptyList())) {
1626                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1627                         if (externalIpCount.getCounter() != 0) {
1628                             return true;
1629                         }
1630                         break;
1631                     }
1632                 }
1633             }
1634         }
1635         return false;
1636     }
1637
1638     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1639             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1640         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1641         try {
1642             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1643             if (address instanceof Inet6Address) {
1644                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1645                 return;
1646             }
1647         } catch (UnknownHostException e) {
1648             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1649             return;
1650         }
1651         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1652         if (leastLoadedExtIpAddr != null) {
1653             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1654             String leastLoadedExtIp = externalIpParts[0];
1655             String leastLoadedExtIpPrefix = externalIpParts[1];
1656             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1657             subnetIp = subnetIpParts[0];
1658             String subnetIpPrefix = subnetIpParts[1];
1659             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1660             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1661                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1662                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1663             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1664
1665
1666             // Check if external IP is already assigned a route. (i.e. External IP is previously
1667             // allocated to any of the subnets)
1668             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1669             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1670             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1671             if (label != null) {
1672                 //update
1673                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1674                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1675                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1676                     label, internalIp, leastLoadedExtIpAddrStr);
1677                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1678                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1679                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1680                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1681                 return;
1682             }
1683
1684             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1685             // for the first time and hence not having a route.
1686             //Get the VPN Name using the network ID
1687             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1688             if (vpnName != null) {
1689                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1690                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1691                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1692                         + "added after gateway-set");
1693                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1694                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1695                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1696                         return;
1697                     }
1698                 }
1699                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1700                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1701                     writeFlowInvTx);
1702             }
1703         }
1704     }
1705
1706     @Nullable
1707     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1708         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1709         for (IpMap ipMap : ipMaps) {
1710             if (ipMap.getExternalIp().equals(externalIp)) {
1711                 if (ipMap.getLabel() != null) {
1712                     return ipMap.getLabel();
1713                 }
1714             }
1715         }
1716         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1717         return null;
1718     }
1719
1720     @Override
1721     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1722         LOG.trace("remove : Router delete method");
1723         /*
1724         ROUTER DELETE SCENARIO
1725         1) Get the router ID from the event.
1726         2) Build the cookie information from the router ID.
1727         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1728         4) Build the flow with the cookie value.
1729         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1730         6) Remove the flows from the other switches which points to the primary and secondary
1731          switches for the flows related the router ID.
1732         7) Get the list of external IP address maintained for the router ID.
1733         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1734         9) Withdraw the corresponding routes from the BGP.
1735          */
1736
1737         if (identifier == null || router == null) {
1738             LOG.error("remove : returning without processing since routers is null");
1739             return;
1740         }
1741
1742         String routerName = router.getRouterName();
1743         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1744             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1745                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1746                         routerName);
1747                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1748                 if (routerId == NatConstants.INVALID_ID) {
1749                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1750                             routerName);
1751                     return;
1752                 }
1753                 long bgpVpnId = NatConstants.INVALID_ID;
1754                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1755                 if (bgpVpnUuid != null) {
1756                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1757                 }
1758                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1759                         tx);
1760                 Uuid networkUuid = router.getNetworkId();
1761
1762                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1763                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1764                     // No NAPT switch for external router, probably because the router is not attached to
1765                     // any
1766                     // internal networks
1767                     LOG.debug(
1768                             "No NAPT switch for router {}, check if router is attached to any internal "
1769                                     + "network",
1770                             routerName);
1771                     return;
1772                 } else {
1773                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1774                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1775                             routerId, tx);
1776                 }
1777                 natOverVxlanUtil.releaseVNI(routerName);
1778             })), NatConstants.NAT_DJC_MAX_RETRIES);
1779     }
1780
1781     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1782                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1783                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1784         LOG.info("handleDisableSnat : Entry");
1785         String routerName = router.getRouterName();
1786         try {
1787             if (routerFlag) {
1788                 removeNaptSwitch(routerName);
1789             } else {
1790                 updateNaptSwitch(routerName, BigInteger.ZERO);
1791             }
1792
1793             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1794             naptManager.removeExternalCounter(routerId);
1795
1796             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1797             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1798                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1799                     + "router ID {} from RouterNaptSwitch model", routerId);
1800                 return;
1801             }
1802             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1803                     networkUuid);
1804             if (extNwProvType == null) {
1805                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1806                 return;
1807             }
1808             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1809             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1810                     externalSubnetList, removeFlowInvTx, extNwProvType);
1811             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1812             String externalSubnetVpn = null;
1813             for (Uuid externalSubnetId : externalSubnetList) {
1814                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1815                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1816                 if (externalSubnet.isPresent()) {
1817                     externalSubnetVpn =  externalSubnetId.getValue();
1818                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1819                             router.getExtGwMacAddress(), removeFlowInvTx);
1820                 }
1821             }
1822             if (externalSubnetVpn == null) {
1823                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1824                         router.getExtGwMacAddress(), removeFlowInvTx);
1825             }
1826             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1827             // for the router ID.
1828             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1829                     + "router ID {} in the DS", routerId);
1830             naptManager.removeMapping(routerId);
1831         } catch (InterruptedException | ExecutionException e) {
1832             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1833         }
1834         LOG.info("handleDisableSnat : Exit");
1835     }
1836
1837     // TODO Clean up the exception handling
1838     @SuppressWarnings("checkstyle:IllegalCatch")
1839     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1840                                              @Nonnull Collection<String> externalIps,
1841                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1842         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1843                 + "with internet vpn {}", routerName, vpnId);
1844         try {
1845             BigInteger naptSwitchDpnId = null;
1846             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1847                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1848             Optional<RouterToNaptSwitch> rtrToNapt;
1849             try {
1850                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1851                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1852             } catch (ReadFailedException e) {
1853                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1854                 rtrToNapt = Optional.absent();
1855             }
1856             if (rtrToNapt.isPresent()) {
1857                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1858             }
1859             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1860
1861             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1862                     writeFlowInvTx);
1863             try {
1864                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1865                 if (extGwMacAddress != null) {
1866                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1867                             + "External Router ID {}", extGwMacAddress, routerId);
1868                 } else {
1869                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1870                             + "External Router ID {}", routerId);
1871                     return;
1872                 }
1873                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1874                         writeFlowInvTx);
1875             } catch (Exception ex) {
1876                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1877                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1878             }
1879             natOverVxlanUtil.releaseVNI(vpnId);
1880         } catch (InterruptedException | ExecutionException e) {
1881             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1882                     + "with internet vpn {}", routerName, vpnId, e);
1883         }
1884         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1885                 routerName, vpnId);
1886     }
1887
1888     // TODO Clean up the exception handling
1889     @SuppressWarnings("checkstyle:IllegalCatch")
1890     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1891         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1892             .setPrimarySwitchId(naptSwitchId).build();
1893         try {
1894             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1895                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1896         } catch (Exception ex) {
1897             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1898                 naptSwitchId, routerName);
1899         }
1900         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1901             naptSwitchId, routerName);
1902     }
1903
1904     protected void removeNaptSwitch(String routerName) {
1905         // Remove router and switch from model
1906         InstanceIdentifier<RouterToNaptSwitch> id =
1907             InstanceIdentifier.builder(NaptSwitches.class)
1908                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1909         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1910         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1911         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1912         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1913     }
1914
1915     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1916                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1917                                                 @Nonnull Collection<String> externalIps,
1918                                                 Collection<Uuid> externalSubnetList,
1919                                                 TypedReadWriteTransaction<Configuration> confTx,
1920                                                 ProviderTypes extNwProvType)
1921             throws InterruptedException, ExecutionException {
1922
1923         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1924         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1925
1926         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1927         // traffic which comes from the  VMs of the NAPT switches)
1928         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1929         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1930
1931         LOG.info(
1932             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1933                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1934         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1935
1936         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1937         // traffic which comes from the VMs of the non NAPT switches)
1938         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1939                 elanManager, idManager, routerId, routerName);
1940         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1941         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1942         LOG.info(
1943             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1944                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1945         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1946
1947         //Remove the flow table 25->44 from NAPT Switch
1948         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1949             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1950         }
1951
1952         //Remove the Outbound flow entry which forwards the packet to FIB Table
1953         LOG.info(
1954             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1955                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1956
1957         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1958             NwConstants.IP_PROT_TCP);
1959         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1960             outboundTcpNatFlowRef);
1961         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1962
1963         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1964             NwConstants.IP_PROT_UDP);
1965         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1966             outboundUdpNatFlowRef);
1967         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1968
1969         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1970             NwConstants.IP_PROT_ICMP);
1971         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1972             icmpDropFlowRef);
1973         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1974         boolean lastRouterOnExternalNetwork =
1975             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1976         if (lastRouterOnExternalNetwork) {
1977             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1978         }
1979         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1980         // External Subnet Vpn Id.
1981         for (Uuid externalSubnetId : externalSubnetList) {
1982             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1983             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1984                 dataBroker, externalSubnetId, routerName, dpnId)) {
1985                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1986                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1987                     natPfibSubnetFlowRef);
1988                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1989                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1990                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1991                     subnetVpnId, dpnId);
1992             }
1993         }
1994
1995         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1996         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1997         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1998
1999         LOG.info(
2000             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2001             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2002         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2003
2004         if (lastRouterOnExternalNetwork) {
2005             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2006             // - This does not work since ext-routers is deleted already - no network info
2007             //Get the VPN ID from the ExternalNetworks model
2008             long vpnId = -1;
2009             if (vpnName == null || vpnName.isEmpty()) {
2010                 // ie called from router delete cases
2011                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2012                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2013                 if (vpnUuid != null) {
2014                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2015                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2016                               + "or disableSNAT scenario", vpnId, networkId);
2017                 }
2018             } else {
2019                 // ie called from disassociate vpn case
2020                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2021                     vpnName);
2022                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2023                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2024                     vpnId);
2025             }
2026
2027             if (vpnId != NatConstants.INVALID_ID) {
2028                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2029                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2030                 FlowEntity natPfibVpnFlowEntity =
2031                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2032                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2033                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2034                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2035             }
2036         }
2037
2038         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2039         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2040         if (ipPortMapping == null) {
2041             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2042             return;
2043         }
2044
2045         for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2046                 Collections.<IntextIpProtocolType>emptyList())) {
2047             for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2048                     Collections.<IpPortMap>emptyList())) {
2049                 String ipPortInternal = ipPortMap.getIpPortInternal();
2050                 String[] ipPortParts = ipPortInternal.split(":");
2051                 if (ipPortParts.length != 2) {
2052                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2053                     return;
2054                 }
2055                 String internalIp = ipPortParts[0];
2056                 String internalPort = ipPortParts[1];
2057
2058                 //Build the flow for the outbound NAPT table
2059                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2060                     + NatConstants.COLON_SEPARATOR + internalPort);
2061                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2062                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2063                 FlowEntity outboundNaptFlowEntity =
2064                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2065
2066                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2067                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2068                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2069
2070                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2071                 String externalIp = ipPortExternal.getIpAddress();
2072                 int externalPort = ipPortExternal.getPortNum();
2073
2074                 //Build the flow for the inbound NAPT table
2075                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2076                     String.valueOf(routerId), externalIp, externalPort);
2077                 FlowEntity inboundNaptFlowEntity =
2078                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2079
2080                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2081                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2082                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2083             }
2084         }
2085     }
2086
2087     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2088                                                     @Nonnull Collection<String> externalIps,
2089                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2090             throws ExecutionException, InterruptedException {
2091         long extVpnId = NatConstants.INVALID_ID;
2092         if (networkId != null) {
2093             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2094             if (vpnUuid != null) {
2095                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2096             } else {
2097                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2098             }
2099         } else {
2100             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2101             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2102         }
2103         if (extVpnId == NatConstants.INVALID_ID) {
2104             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2105             extVpnId = routerId;
2106         }
2107         for (String ip : externalIps) {
2108             String extIp = removeMaskFromIp(ip);
2109             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2110             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2111                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2112                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2113             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2114             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2115         }
2116     }
2117
2118     private String removeMaskFromIp(String ip) {
2119         if (ip != null && !ip.trim().isEmpty()) {
2120             return ip.split("/")[0];
2121         }
2122         return ip;
2123     }
2124
2125     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2126                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2127                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2128             throws ExecutionException, InterruptedException {
2129         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2130         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2131
2132         //Remove the NAPT PFIB TABLE entry
2133         long vpnId = -1;
2134         if (vpnName != null) {
2135             // ie called from disassociate vpn case
2136             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2137                     + "with vpnName {}", vpnName);
2138             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2139             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2140                     vpnId);
2141         }
2142
2143         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2144                 networkId, routerName, dpnId)) {
2145             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2146             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2147             FlowEntity natPfibVpnFlowEntity =
2148                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2149             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2150                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2151             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2152
2153             // Remove IP-PORT active NAPT entries and release port from IdManager
2154             // For the router ID get the internal IP , internal port and the corresponding
2155             // external IP and external Port.
2156             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2157             if (ipPortMapping == null) {
2158                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2159                 return;
2160             }
2161             for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2162                     Collections.<IntextIpProtocolType>emptyList())) {
2163                 for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2164                         Collections.<IpPortMap>emptyList())) {
2165                     String ipPortInternal = ipPortMap.getIpPortInternal();
2166                     String[] ipPortParts = ipPortInternal.split(":");
2167                     if (ipPortParts.length != 2) {
2168                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2169                                 + "and port");
2170                         return;
2171                     }
2172                     String internalIp = ipPortParts[0];
2173                     String internalPort = ipPortParts[1];
2174
2175                     //Build the flow for the outbound NAPT table
2176                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2177                             + NatConstants.COLON_SEPARATOR + internalPort);
2178                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2179                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2180                     FlowEntity outboundNaptFlowEntity =
2181                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2182
2183                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2184                             + "active switch with the DPN ID {} and router ID {}",
2185                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2186                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2187
2188                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2189                     String externalIp = ipPortExternal.getIpAddress();
2190                     int externalPort = ipPortExternal.getPortNum();
2191
2192                     //Build the flow for the inbound NAPT table
2193                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2194                         String.valueOf(routerId), externalIp, externalPort);
2195                     FlowEntity inboundNaptFlowEntity =
2196                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2197
2198                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2199                             + "active active switch with the DPN ID {} and router ID {}",
2200                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2201                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2202
2203                     // Finally release port from idmanager
2204                     String internalIpPort = internalIp + ":" + internalPort;
2205                     naptManager.removePortFromPool(internalIpPort, externalIp);
2206
2207                     //Remove sessions from models
2208                     naptManager.removeIpPortMappingForRouterID(routerId);
2209                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2210                 }
2211             }
2212         } else {
2213             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2214         }
2215     }
2216
2217     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2218             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2219             throws ExecutionException, InterruptedException {
2220         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2221
2222         // Remove the flows from the other switches which points to the primary and secondary switches
2223         // for the flows related the router ID.
2224         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2225         if (allSwitchList.isEmpty()) {
2226             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2227             return;
2228         }
2229         for (BigInteger dpnId : allSwitchList) {
2230             if (!naptSwitchDpnId.equals(dpnId)) {
2231                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2232
2233                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2234                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2235                 FlowEntity preSnatFlowEntity =
2236                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2237
2238                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2239                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2240                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2241
2242                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2243                 long groupId = createGroupId(getGroupIdKey(routerName));
2244
2245                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2246                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2247                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2248             }
2249         }
2250     }
2251
2252     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2253                                          @Nonnull Collection<String> externalIps, @Nullable String vpnName,
2254                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2255             throws ExecutionException, InterruptedException {
2256         //Withdraw the corresponding routes from the BGP.
2257         //Get the network ID using the router ID.
2258         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2259                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2260         if (networkUuid == null) {
2261             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2262             return;
2263         }
2264
2265         if (externalIps.isEmpty()) {
2266             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2267             return;
2268         }
2269
2270         if (vpnName == null) {
2271             //Get the VPN Name using the network ID
2272             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2273             if (vpnName == null) {
2274                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2275                     networkUuid, routerId);
2276                 return;
2277             }
2278         }
2279         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2280
2281         //Remove custom FIB routes
2282         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2283         for (String extIp : externalIps) {
2284             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2285         }
2286     }
2287
2288     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2289                                             final Uuid networkUuid, String extGwMacAddress,
2290                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2291             throws ExecutionException, InterruptedException {
2292         clearBgpRoutes(extIp, vpnName);
2293         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2294                 removeFlowInvTx);
2295     }
2296
2297     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2298                                              String vpnName, Uuid extNetworkId, long tempLabel,
2299                                              String gwMacAddress, boolean switchOver,
2300                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2301             throws ExecutionException, InterruptedException {
2302         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2303         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2304         if (routerName == null) {
2305             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2306             return;
2307         }
2308         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2309         if (extNwProvType == null) {
2310             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2311             return;
2312         }
2313         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2314          * external network provided type is VxLAN
2315          */
2316         if (extNwProvType == ProviderTypes.VXLAN) {
2317             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2318             );
2319             return;
2320         }
2321         if (tempLabel < 0) {
2322             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2323                     extIp, routerId);
2324             return;
2325         }
2326         final long label = tempLabel;
2327         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2328         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2329             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2330             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2331             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2332                     extRouter);
2333
2334             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2335                     externalSubnetId);
2336
2337             if (externalSubnet.isPresent()) {
2338                 vpnName =  externalSubnetId.getValue();
2339             }
2340         }
2341         final String externalVpn = vpnName;
2342         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2343                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2344                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2345         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2346
2347         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2348         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2349         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2350             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2351             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2352         }
2353         if (!switchOver) {
2354             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2355                 Futures.transformAsync(future, result -> {
2356                     //Release label
2357                     if (result.isSuccessful()) {
2358                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2359                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2360                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2361                         return vpnService.removeVpnLabel(labelInput);
2362                     } else {
2363                         String errMsg =
2364                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2365                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2366                         LOG.error(errMsg);
2367                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2368                     }
2369                 }, MoreExecutors.directExecutor());
2370
2371             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2372
2373                 @Override
2374                 public void onFailure(@Nonnull Throwable error) {
2375                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2376                         + "got external ip {}", label, extIp, error);
2377                 }
2378
2379                 @Override
2380                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2381                     if (result.isSuccessful()) {
2382                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2383                             + "from VPN {}", externalIp, externalVpn);
2384                     } else {
2385                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2386                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2387                     }
2388                 }
2389             }, MoreExecutors.directExecutor());
2390         } else {
2391             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2392                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2393         }
2394     }
2395
2396     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2397                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2398                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2399             throws ExecutionException, InterruptedException {
2400         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2401         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2402         if (routerName == null) {
2403             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2404             return;
2405         }
2406         //Get the external network provider type from networkId
2407         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2408         if (extNwProvType == null) {
2409             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2410                     networkUuid);
2411             return;
2412         }
2413         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2414          *  external network provided type is VxLAN
2415          */
2416         if (extNwProvType == ProviderTypes.VXLAN) {
2417             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2418             return;
2419         }
2420         //Get IPMaps from the DB for the router ID
2421         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2422         if (dbIpMaps.isEmpty()) {
2423             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2424             return;
2425         }
2426
2427         long tempLabel = NatConstants.INVALID_ID;
2428         for (IpMap dbIpMap : dbIpMaps) {
2429             String dbExternalIp = dbIpMap.getExternalIp();
2430             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2431             //Select the IPMap, whose external IP is the IP for which FIB is installed
2432             if (extIp.equals(dbExternalIp)) {
2433                 tempLabel = dbIpMap.getLabel();
2434                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2435                     tempLabel, dbExternalIp, routerId);
2436                 break;
2437             }
2438         }
2439         if (tempLabel == NatConstants.INVALID_ID) {
2440             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2441                     extIp, routerId);
2442             return;
2443         }
2444
2445         final long label = tempLabel;
2446         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2447         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2448                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2449                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2450         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2451
2452         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2453         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2454         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2455             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2456             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2457         }
2458         if (!switchOver) {
2459             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2460                     Futures.transformAsync(future, result -> {
2461                         //Release label
2462                         if (result.isSuccessful()) {
2463                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2464                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2465                             return vpnService.removeVpnLabel(labelInput);
2466                         } else {
2467                             String errMsg =
2468                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2469                                             + "prefix %s Failed - %s",
2470                                             dpnId, externalIp, result.getErrors());
2471                             LOG.error(errMsg);
2472                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2473                         }
2474                     }, MoreExecutors.directExecutor());
2475
2476             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2477
2478                 @Override
2479                 public void onFailure(@Nonnull Throwable error) {
2480                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2481                 }
2482
2483                 @Override
2484                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2485                     if (result.isSuccessful()) {
2486                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2487                             + "from VPN {}", externalIp, vpnName);
2488                     } else {
2489                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2490                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2491                     }
2492                 }
2493             }, MoreExecutors.directExecutor());
2494         } else {
2495             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2496                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2497         }
2498     }
2499
2500     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2501             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2502             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2503         //Withdraw the corresponding routes from the BGP.
2504         //Get the network ID using the router ID.
2505         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2506                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2507         if (networkUuid == null) {
2508             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2509             return;
2510         }
2511
2512         if (externalIps == null || externalIps.isEmpty()) {
2513             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2514             return;
2515         }
2516
2517         if (vpnName == null) {
2518             //Get the VPN Name using the network ID
2519             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2520             if (vpnName == null) {
2521                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2522                     networkUuid, routerId);
2523                 return;
2524             }
2525         }
2526         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2527
2528         //Remove custom FIB routes
2529         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2530         for (String extIp : externalIps) {
2531             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2532                     writeFlowInvTx);
2533         }
2534     }
2535
2536     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2537         //Inform BGP about the route removal
2538         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2539         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2540         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2541     }
2542
2543     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2544             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2545         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2546         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2547             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2548         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2549     }
2550
2551     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2552             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2553         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2554         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2555         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2556         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2557     }
2558
2559     /**
2560      * router association to vpn.
2561      *
2562      * @param routerName - Name of router
2563      * @param routerId - router id
2564      * @param bgpVpnName BGP VPN name
2565      */
2566     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2567             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2568         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2569         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2570             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2571
2572             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2573
2574             if (bgpVpnId != NatConstants.INVALID_ID) {
2575                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2576                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2577                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2578                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2579                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2580                     getRoutersIdentifier(bgpVpnId), rtrs);
2581
2582                 // Get the allocated Primary NAPT Switch for this router
2583                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2584
2585                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2586                         routerId, bgpVpnId);
2587                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2588
2589                 // Get the group ID
2590                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2591                 createGroupId(getGroupIdKey(routerName));
2592                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2593                         extNwProvType);
2594             }
2595         }
2596     }
2597
2598     /**
2599      * router disassociation from vpn.
2600      *
2601      * @param routerName - Name of router
2602      * @param routerId - router id
2603      * @param bgpVpnName BGP VPN name
2604      */
2605     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2606             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2607         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2608         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2609             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2610             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2611
2612             // Get the allocated Primary NAPT Switch for this router
2613             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2614
2615             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2616             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2617
2618             // Get the group ID
2619             createGroupId(getGroupIdKey(routerName));
2620             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2621             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2622                     writeFlowInvTx, extNwProvType);
2623         }
2624     }
2625
2626     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2627         InstanceIdentifier<Routers> routerInstanceIndentifier =
2628             InstanceIdentifier.builder(ExtRouters.class)
2629                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2630         try {
2631             Optional<Routers> routerData = SingleTransactionDataBroker
2632                     .syncReadOptional(dataBroker,
2633                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2634             return routerData.isPresent() && routerData.get().isEnableSnat();
2635         } catch (ReadFailedException e) {
2636             LOG.error("Failed to read data for router id {}", routerUuid, e);
2637             return false;
2638         }
2639     }
2640
2641     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2642         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2643
2644         long changedVpnId = bgpVpnId;
2645         String idType = "BGP VPN";
2646         if (bgpVpnId == NatConstants.INVALID_ID) {
2647             changedVpnId = routerId;
2648             idType = "router";
2649         }
2650
2651         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2652         if (switches.isEmpty()) {
2653             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2654             return;
2655         }
2656         for (BigInteger dpnId : switches) {
2657             // Update the BGP VPN ID in the SNAT miss entry to group
2658             if (!dpnId.equals(primarySwitchId)) {
2659                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2660                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2661                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2662                 long groupId = createGroupId(getGroupIdKey(routerName));
2663                 if (!isSnatCfgd) {
2664                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2665                 }
2666
2667                 LOG.debug(
2668                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2669                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2670                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2671                 mdsalManager.addFlow(confTx, flowEntity);
2672             } else {
2673                 LOG.debug(
2674                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2675                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2676                 FlowEntity flowEntity =
2677                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2678                 mdsalManager.addFlow(confTx, flowEntity);
2679
2680                 LOG.debug(
2681                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2682                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2683                         idType, changedVpnId, primarySwitchId);
2684                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2685                         changedVpnId, confTx, extNwProvType);
2686
2687                 LOG.debug(
2688                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2689                                 + "which punts the packet to the controller in the Primary switch {}",
2690                         idType, changedVpnId, primarySwitchId);
2691                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2692
2693                 LOG.debug(
2694                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2695                                 + " outgoing packet to FIB Table in the Primary switch {}",
2696                         idType, changedVpnId, primarySwitchId);
2697                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2698
2699                 LOG.debug(
2700                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2701                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2702                                 + " {}", idType, changedVpnId, primarySwitchId);
2703                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2704
2705                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2706                         primarySwitchId);
2707                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2708                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2709                 if (vpnId != NatConstants.INVALID_ID) {
2710                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2711                 }
2712             }
2713         }
2714     }
2715
2716     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2717         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2718         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2719         if (ipPortMapping == null) {
2720             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2721             return;
2722         }
2723         // Get the External Gateway MAC Address
2724         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2725         if (extGwMacAddress != null) {
2726             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2727                     extGwMacAddress, routerId);
2728         } else {
2729             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2730                     routerId);
2731             return;
2732         }
2733         for (IntextIpProtocolType intextIpProtocolType : requireNonNullElse(ipPortMapping.getIntextIpProtocolType(),
2734                 Collections.<IntextIpProtocolType>emptyList())) {
2735             for (IpPortMap ipPortMap : requireNonNullElse(intextIpProtocolType.getIpPortMap(),
2736                     Collections.<IpPortMap>emptyList())) {
2737                 String ipPortInternal = ipPortMap.getIpPortInternal();
2738                 String[] ipPortParts = ipPortInternal.split(":");
2739                 if (ipPortParts.length != 2) {
2740                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2741                     return;
2742                 }
2743                 String internalIp = ipPortParts[0];
2744                 String internalPort = ipPortParts[1];
2745                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2746                         internalIp, internalPort);
2747                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2748                 NAPTEntryEvent.Protocol protocol;
2749                 switch (protocolTypes) {
2750                     case TCP:
2751                         protocol = NAPTEntryEvent.Protocol.TCP;
2752                         break;
2753                     case UDP:
2754                         protocol = NAPTEntryEvent.Protocol.UDP;
2755                         break;
2756                     default:
2757                         protocol = NAPTEntryEvent.Protocol.TCP;
2758                 }
2759                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2760                 SessionAddress externalAddress =
2761                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2762                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2763                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2764                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2765                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2766                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2767             }
2768         }
2769     }
2770
2771     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2772                                                           long changedVpnId) {
2773
2774         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2775             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2776         List<MatchInfo> matches = new ArrayList<>();
2777         matches.add(MatchEthernetType.IPV4);
2778         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2779
2780         List<ActionInfo> actionsInfo = new ArrayList<>();
2781         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2782                 elanManager, idManager, changedVpnId, routerName);
2783         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2784         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2785                 actionsInfo);
2786         actionsInfo.add(new ActionGroup(groupId));
2787         List<InstructionInfo> instructions = new ArrayList<>();
2788         instructions.add(new InstructionApplyActions(actionsInfo));
2789         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2790         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2791             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2792             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2793
2794         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2795         return flowEntity;
2796     }
2797
2798     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2799                                                                         long changedVpnId) {
2800
2801         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2802                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2803         List<MatchInfo> matches = new ArrayList<>();
2804         matches.add(MatchEthernetType.IPV4);
2805         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2806
2807         List<InstructionInfo> instructions = new ArrayList<>();
2808         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2809
2810         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2811         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2812             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2813             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2814
2815         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2816         return flowEntity;
2817     }
2818
2819     // TODO : Replace this with ITM Rpc once its available with full functionality
2820     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2821         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2822
2823         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2824             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2825         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2826                 extNwProvType);
2827         mdsalManager.addFlow(confTx, flowEntity);
2828     }
2829
2830     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2831             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2832         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2833             dpId, routerName, changedVpnId);
2834         List<MatchInfo> matches = new ArrayList<>();
2835         matches.add(MatchEthernetType.IPV4);
2836
2837         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2838         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2839             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2840         }
2841         matches.add(new MatchTunnelId(tunnelId));
2842
2843         List<InstructionInfo> instructions = new ArrayList<>();
2844         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2845             MetaDataUtil.METADATA_MASK_VRFID));
2846         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2847         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2848         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2849         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2850             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2851             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2852         return flowEntity;
2853     }
2854
2855     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2856                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2857         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2858             dpnId, routerId, changedVpnId);
2859         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2860                 NwConstants.IP_PROT_TCP);
2861         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2862         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2863
2864         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2865                 NwConstants.IP_PROT_UDP);
2866         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2867         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2868
2869         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2870         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2871         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2872     }
2873
2874     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2875                                                            long changedVpnId, int protocol) {
2876         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2877             dpId, routerId, changedVpnId);
2878         BigInteger cookie = getCookieOutboundFlow(routerId);
2879         List<MatchInfo> matches = new ArrayList<>();
2880         matches.add(MatchEthernetType.IPV4);
2881         matches.add(new MatchIpProtocol((short)protocol));
2882         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2883
2884         List<InstructionInfo> instructions = new ArrayList<>();
2885         List<ActionInfo> actionsInfos = new ArrayList<>();
2886         actionsInfos.add(new ActionPuntToController());
2887         if (snatPuntTimeout != 0) {
2888             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2889         }
2890         instructions.add(new InstructionApplyActions(actionsInfos));
2891
2892         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2893         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2894             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2895         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2896         return flowEntity;
2897     }
2898
2899     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2900                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2901         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2902             dpnId, segmentId, changedVpnId);
2903         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2904         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2905     }
2906
2907     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2908
2909         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2910             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2911         List<MatchInfo> matches = new ArrayList<>();
2912         matches.add(MatchEthernetType.IPV4);
2913         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2914
2915         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2916         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2917         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2918         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2919         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2920
2921         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2922         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2923             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2924             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2925         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2926         return flowEntity;
2927     }
2928
2929     @Override
2930     protected ExternalRoutersListener getDataTreeChangeListener() {
2931         return ExternalRoutersListener.this;
2932     }
2933
2934     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2935                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2936         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2937                 routerName);
2938         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2939             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2940             if (subnetVpnId != -1) {
2941                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2942                     + "and vpnId {}", dpnId, subnetVpnId);
2943                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2944             }
2945         }
2946     }
2947 }