natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  85 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  86 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  87 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 150 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 151 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 152 import org.opendaylight.yangtools.yang.common.RpcResult;
 153 import org.slf4j.Logger;
 154 import org.slf4j.LoggerFactory;
 155
 156
 157 @Singleton
 158 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 159     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 160
 161     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 162     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 163
 164     private final DataBroker dataBroker;
 165     private final ManagedNewTransactionRunner txRunner;
 166     private final IMdsalApiManager mdsalManager;
 167     private final ItmRpcService itmManager;
 168     private final OdlInterfaceRpcService odlInterfaceRpcService;
 169     private final IdManagerService idManager;
 170     private final NaptManager naptManager;
 171     private final NAPTSwitchSelector naptSwitchSelector;
 172     private final IBgpManager bgpManager;
 173     private final VpnRpcService vpnService;
 174     private final FibRpcService fibService;
 175     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 176     private final NaptEventHandler naptEventHandler;
 177     private final NaptPacketInHandler naptPacketInHandler;
 178     private final IFibManager fibManager;
 179     private final IVpnManager vpnManager;
 180     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 181     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 182     private final NatMode natMode;
 183     private final INeutronVpnManager nvpnManager;
 184     private final IElanService elanManager;
 185     private final JobCoordinator coordinator;
 186     private final UpgradeState upgradeState;
 187     private final IInterfaceManager interfaceManager;
 188     private final NatOverVxlanUtil natOverVxlanUtil;
 189     private final int snatPuntTimeout;
 190
 191     @Inject
 192     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 193                                    final ItmRpcService itmManager,
 194                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 195                                    final IdManagerService idManager,
 196                                    final NaptManager naptManager,
 197                                    final NAPTSwitchSelector naptSwitchSelector,
 198                                    final IBgpManager bgpManager,
 199                                    final VpnRpcService vpnService,
 200                                    final FibRpcService fibService,
 201                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 202                                    final NaptEventHandler naptEventHandler,
 203                                    final NaptPacketInHandler naptPacketInHandler,
 204                                    final IFibManager fibManager,
 205                                    final IVpnManager vpnManager,
 206                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 207                                    final INeutronVpnManager nvpnManager,
 208                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 209                                    final NatserviceConfig config,
 210                                    final IElanService elanManager,
 211                                    final JobCoordinator coordinator,
 212                                    final UpgradeState upgradeState,
 213                                    final NatOverVxlanUtil natOverVxlanUtil,
 214                                    final IInterfaceManager interfaceManager) {
 215         super(Routers.class, ExternalRoutersListener.class);
 216         this.dataBroker = dataBroker;
 217         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 218         this.mdsalManager = mdsalManager;
 219         this.itmManager = itmManager;
 220         this.odlInterfaceRpcService = odlInterfaceRpcService;
 221         this.idManager = idManager;
 222         this.naptManager = naptManager;
 223         this.naptSwitchSelector = naptSwitchSelector;
 224         this.bgpManager = bgpManager;
 225         this.vpnService = vpnService;
 226         this.fibService = fibService;
 227         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 228         this.naptEventHandler = naptEventHandler;
 229         this.naptPacketInHandler = naptPacketInHandler;
 230         this.fibManager = fibManager;
 231         this.vpnManager = vpnManager;
 232         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 233         this.nvpnManager = nvpnManager;
 234         this.elanManager = elanManager;
 235         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 236         this.coordinator = coordinator;
 237         this.upgradeState = upgradeState;
 238         this.interfaceManager = interfaceManager;
 239         this.natOverVxlanUtil = natOverVxlanUtil;
 240         if (config != null) {
 241             this.natMode = config.getNatMode();
 242             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 243         } else {
 244             this.natMode = NatMode.Controller;
 245             this.snatPuntTimeout = 0;
 246         }
 247     }
 248
 249     @Override
 250     @PostConstruct
 251     public void init() {
 252         LOG.info("{} init", getClass().getSimpleName());
 253         // This class handles ExternalRouters for Controller SNAT mode.
 254         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 255         if (natMode == NatMode.Controller) {
 256             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 257             NatUtil.createGroupIdPool(idManager);
 258         }
 259     }
 260
 261     @Override
 262     protected InstanceIdentifier<Routers> getWildCardPath() {
 263         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 264     }
 265
 266     @Override
 267     // TODO Clean up the exception handling
 268     @SuppressWarnings("checkstyle:IllegalCatch")
 269     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 270         // Populate the router-id-name container
 271         String routerName = routers.getRouterName();
 272         LOG.info("add : external router event for {}", routerName);
 273         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 274         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 275         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 276         try {
 277             if (routers.isEnableSnat()) {
 278                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 279                     () -> Collections.singletonList(
 280                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 281                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 282                             long bgpVpnId = NatConstants.INVALID_ID;
 283                             if (bgpVpnUuid != null) {
 284                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 285                             }
 286                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 287                             // Allocate Primary Napt Switch for this router
 288                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 289                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 290                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 291                             }
 292                         }
 293                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 294             } else {
 295                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 296             }
 297         } catch (Exception ex) {
 298             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 299                     routerName, ex);
 300         }
 301     }
 302
 303     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 304                                  TypedWriteTransaction<Configuration> confTx) {
 305         String routerName = routers.getRouterName();
 306         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 307
 308         naptManager.initialiseExternalCounter(routers, routerId);
 309         subnetRegisterMapping(routers, routerId);
 310
 311         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 312             primarySwitchId, routerName);
 313
 314         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 315                 routers.getNetworkId());
 316         if (extNwProvType == null) {
 317             LOG.error("handleEnableSnat : External Network Provider Type missing");
 318             return;
 319         }
 320
 321         if (bgpVpnId != NatConstants.INVALID_ID) {
 322             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 323                     extNwProvType);
 324         } else {
 325             // write metadata and punt
 326             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 327             // Now install entries in SNAT tables to point to Primary for each router
 328             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 329             for (BigInteger dpnId : switches) {
 330                 // Handle switches and NAPT switches separately
 331                 if (!dpnId.equals(primarySwitchId)) {
 332                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 333                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 334                 } else {
 335                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 336                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 337                 }
 338             }
 339         }
 340
 341         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 342         if (externalIps.isEmpty()) {
 343             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 344             return;
 345         } else {
 346             for (String externalIpAddrPrefix : externalIps) {
 347                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 348                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 349                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 350                 );
 351             }
 352         }
 353         LOG.debug("handleEnableSnat : Exit");
 354     }
 355
 356     private BigInteger getPrimaryNaptSwitch(String routerName) {
 357         // Allocate Primary Napt Switch for this router
 358         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 359         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 360             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 361                 primarySwitchId, routerName);
 362             return primarySwitchId;
 363         }
 364         // Allocated an id from VNI pool for the Router.
 365         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 366         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 367         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 368
 369         return primarySwitchId;
 370     }
 371
 372     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 373                                                      TypedWriteTransaction<Configuration> confTx) {
 374         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 375         if (extVpnId == NatConstants.INVALID_ID) {
 376             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 377             extVpnId = routerId;
 378         }
 379         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 380         if (externalIps.isEmpty()) {
 381             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 382                 dpnId, extVpnId);
 383             return;
 384         }
 385         for (String ip : externalIps) {
 386             Uuid subnetId = getSubnetIdForFixedIp(ip);
 387             if (subnetId != null) {
 388                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 389                 if (subnetVpnId != NatConstants.INVALID_ID) {
 390                     extVpnId = subnetVpnId;
 391                 }
 392                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 393                     dpnId, extVpnId, subnetId);
 394                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 395                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 396             }
 397         }
 398     }
 399
 400     @Nullable
 401     private Uuid getSubnetIdForFixedIp(String ip) {
 402         if (ip != null) {
 403             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 404             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 405             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 406         }
 407         LOG.error("getSubnetIdForFixedIp : ip is null");
 408         return null;
 409     }
 410
 411     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 412         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 413         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 414         int counter = 0;
 415         int extIpCounter = externalIps.size();
 416         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 417                 counter, extIpCounter);
 418         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 419         if (subnetIds == null) {
 420             return;
 421         }
 422         for (Uuid subnet : subnetIds) {
 423             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 424             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 425                 .builder(Subnetmaps.class)
 426                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 427                 .build();
 428             Optional<Subnetmap> sn;
 429             try {
 430                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 431                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 432             } catch (ReadFailedException e) {
 433                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 434                 sn = Optional.absent();
 435             }
 436             if (sn.isPresent()) {
 437                 // subnets
 438                 Subnetmap subnetmapEntry = sn.get();
 439                 String subnetString = subnetmapEntry.getSubnetIp();
 440                 String[] subnetSplit = subnetString.split("/");
 441                 String subnetIp = subnetSplit[0];
 442                 try {
 443                     InetAddress address = InetAddress.getByName(subnetIp);
 444                     if (address instanceof Inet6Address) {
 445                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 446                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 447                         continue;
 448                     }
 449                 } catch (UnknownHostException e) {
 450                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 451                     return;
 452                 }
 453                 String subnetPrefix = "0";
 454                 if (subnetSplit.length == 2) {
 455                     subnetPrefix = subnetSplit[1];
 456                 }
 457                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 458                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 459                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 460                 //externalIps
 461                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 462                         counter, extIpCounter);
 463                 if (extIpCounter != 0) {
 464                     if (counter < extIpCounter) {
 465                         String[] ipSplit = externalIps.get(counter).split("/");
 466                         String externalIp = ipSplit[0];
 467                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 468                         if (ipSplit.length == 2) {
 469                             extPrefix = ipSplit[1];
 470                         }
 471                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 472                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 473                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 474                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 475                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 476                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 477                     } else {
 478                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 479                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 480                         String[] ipSplit = externalIps.get(counter).split("/");
 481                         String externalIp = ipSplit[0];
 482                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 483                         if (ipSplit.length == 2) {
 484                             extPrefix = ipSplit[1];
 485                         }
 486                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 487                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 488                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 489                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 490                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 491                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 492                             externalIp, extPrefix);
 493                     }
 494                 }
 495                 counter++;
 496                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 497             } else {
 498                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 499             }
 500         }
 501     }
 502
 503     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 504             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 505             throws ExecutionException, InterruptedException {
 506         //Check if BGP VPN exists. If exists then invoke the new method.
 507         if (bgpVpnId != NatConstants.INVALID_ID) {
 508             if (bgpVpnUuid != null) {
 509                 String bgpVpnName = bgpVpnUuid.getValue();
 510                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 511                     bgpVpnId, bgpVpnName);
 512                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 513                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 514                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 515             }
 516             if (create) {
 517                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 518             } else {
 519                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 520             }
 521             return;
 522         }
 523
 524         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 525         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 526     }
 527
 528     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 529             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 530         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 531         if (switches.isEmpty()) {
 532             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 533             return;
 534         }
 535         if (routerId == NatConstants.INVALID_ID) {
 536             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 537                     + "default NAT route in FIB", routerName);
 538             return;
 539         }
 540         for (BigInteger dpnId : switches) {
 541             if (create) {
 542                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 543                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 544                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 545             } else {
 546                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 547                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 548                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 549             }
 550         }
 551     }
 552
 553     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 554         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 555         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 556         if (dpnIds.isEmpty()) {
 557             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 558                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 559             return;
 560         }
 561         for (BigInteger dpnId : dpnIds) {
 562             if (bgpVpnId != NatConstants.INVALID_ID) {
 563                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 564                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 565                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 566             } else {
 567                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 568                     + "in dpn {} for the internal vpn", routerId, dpnId);
 569                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 570             }
 571         }
 572     }
 573
 574     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 575             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 576             throws ExecutionException, InterruptedException {
 577         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 578         if (dpnIds.isEmpty()) {
 579             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 580                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 581             return;
 582         }
 583         for (BigInteger dpnId : dpnIds) {
 584             if (bgpVpnId != NatConstants.INVALID_ID) {
 585                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 586                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 587                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 588             } else {
 589                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 590                     + "in dpn {} for the internal vpn", routerId, dpnId);
 591                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 592             }
 593         }
 594     }
 595
 596     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 597         TypedWriteTransaction<Configuration> confTx) {
 598         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 599         if (routerId != NatConstants.INVALID_ID) {
 600             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 601                     primarySwitchId, routerId);
 602             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 603         } else {
 604             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 605                 + "createAndInstallMissEntry", routerName);
 606         }
 607     }
 608
 609     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 610         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 611                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 612     }
 613
 614     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 615         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 616                 .FLOWID_SEPARATOR + vpnId;
 617     }
 618
 619     public BigInteger getCookieOutboundFlow(long routerId) {
 620         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 621             BigInteger.valueOf(routerId));
 622     }
 623
 624     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 625         long l4SrcPortField;
 626         long l4DstPortField;
 627         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 628
 629         if (protocol == NwConstants.IP_PROT_TCP) {
 630             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 631             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 632         } else {
 633             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 634             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 635         }
 636         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 637                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 638                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 639                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 640                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 641                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 642                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 643                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 644                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 645                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 646                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 647                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 648                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 649                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 650                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 651
 652         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 653                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 654     }
 655
 656     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 657         List<MatchInfo> matches = new ArrayList<>();
 658         matches.add(MatchEthernetType.IPV4);
 659         matches.add(MatchIpProtocol.ICMP);
 660         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 661
 662         List<ActionInfo> actionInfos = new ArrayList<>();
 663         actionInfos.add(new ActionDrop());
 664
 665         List<InstructionInfo> instructions = new ArrayList<>();
 666         instructions.add(new InstructionApplyActions(actionInfos));
 667
 668         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 669                 NwConstants.IP_PROT_ICMP);
 670         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 671                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 672                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 673         return flow;
 674     }
 675
 676     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 677         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 678         BigInteger cookie = getCookieOutboundFlow(routerId);
 679         List<MatchInfo> matches = new ArrayList<>();
 680         matches.add(MatchEthernetType.IPV4);
 681         matches.add(new MatchIpProtocol((short)protocol));
 682         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 683
 684         List<InstructionInfo> instructions = new ArrayList<>();
 685         List<ActionInfo> actionsInfos = new ArrayList<>();
 686         actionsInfos.add(new ActionPuntToController());
 687         if (snatPuntTimeout != 0) {
 688             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 689         }
 690         instructions.add(new InstructionApplyActions(actionsInfos));
 691
 692         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 693         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 694             5, flowRef, 0, 0,
 695             cookie, matches, instructions);
 696         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 697         return flowEntity;
 698     }
 699
 700     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 701         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 702         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 703         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 704         mdsalManager.addFlow(confTx, tcpflowEntity);
 705
 706         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 707         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 708         mdsalManager.addFlow(confTx, udpflowEntity);
 709
 710         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 711         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 712         mdsalManager.addFlow(confTx, icmpDropFlow);
 713     }
 714
 715     @Nullable
 716     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 717         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 718         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 719         try {
 720             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 721                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 722                     .setSourceDpid(srcDpId)
 723                     .setDestinationDpid(dstDpId)
 724                     .setTunnelType(tunType)
 725                     .build());
 726             rpcResult = result.get();
 727             if (!rpcResult.isSuccessful()) {
 728                 tunType = TunnelTypeGre.class;
 729                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 730                     .setSourceDpid(srcDpId)
 731                     .setDestinationDpid(dstDpId)
 732                     .setTunnelType(tunType)
 733                     .build());
 734                 rpcResult = result.get();
 735                 if (!rpcResult.isSuccessful()) {
 736                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 737                             rpcResult.getErrors());
 738                 } else {
 739                     return rpcResult.getResult().getInterfaceName();
 740                 }
 741                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 742                         rpcResult.getErrors());
 743             } else {
 744                 return rpcResult.getResult().getInterfaceName();
 745             }
 746         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 747             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 748                     + "between {} and {}", srcDpId, dstDpId, e);
 749         }
 750
 751         return null;
 752     }
 753
 754     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 755         TypedWriteTransaction<Configuration> confTx) {
 756
 757         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 758         // Install miss entry pointing to group
 759         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 760         mdsalManager.addFlow(confTx, flowEntity);
 761     }
 762
 763     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 764             String routerName, long routerId) {
 765         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 766             dpnId, bucketInfo.get(0));
 767         // Install the select group
 768         long groupId = createGroupId(getGroupIdKey(routerName));
 769         GroupEntity groupEntity =
 770             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 771         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 772         mdsalManager.syncInstallGroup(groupEntity);
 773         // Install miss entry pointing to group
 774         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 775         if (flowEntity == null) {
 776             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 777                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 778                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 779             return;
 780         }
 781         mdsalManager.installFlow(flowEntity);
 782     }
 783
 784     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 785         long groupId = createGroupId(getGroupIdKey(routerName));
 786         GroupEntity groupEntity =
 787             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 788         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 789         mdsalManager.syncInstallGroup(groupEntity);
 790         return groupId;
 791     }
 792
 793     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 794         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 795                 dpId, routerName, groupId);
 796         List<MatchInfo> matches = new ArrayList<>();
 797         matches.add(MatchEthernetType.IPV4);
 798         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 799
 800         List<ActionInfo> actionsInfo = new ArrayList<>();
 801         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 802                 routerId, routerName);
 803         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 804         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 805         actionsInfo.add(new ActionGroup(groupId));
 806         List<InstructionInfo> instructions = new ArrayList<>();
 807         instructions.add(new InstructionApplyActions(actionsInfo));
 808         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 809         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 810                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 811                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 812
 813         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 814         return flowEntity;
 815     }
 816
 817     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 818
 819         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 820             routerName);
 821         List<MatchInfo> matches = new ArrayList<>();
 822         matches.add(MatchEthernetType.IPV4);
 823         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 824
 825         List<InstructionInfo> instructions = new ArrayList<>();
 826         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 827
 828         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 829         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 830             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 831             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 832
 833         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 834         return flowEntity;
 835     }
 836
 837     // TODO : Replace this with ITM Rpc once its available with full functionality
 838     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 839         TypedWriteTransaction<Configuration> confTx) {
 840
 841         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 842             dpnId, routerName);
 843         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 844         if (flowEntity == null) {
 845             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 846                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 847                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 848                     dpnId, routerName);
 849             return;
 850         }
 851         mdsalManager.addFlow(confTx, flowEntity);
 852
 853     }
 854
 855     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 856         List<MatchInfo> matches = new ArrayList<>();
 857         matches.add(MatchEthernetType.IPV4);
 858         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 859                 idManager, routerId, routerName);
 860         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 861         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 862         List<InstructionInfo> instructions = new ArrayList<>();
 863         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 864                 MetaDataUtil.METADATA_MASK_VRFID));
 865         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 866         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 867                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 868                 instructions);
 869         return flowEntity;
 870     }
 871
 872     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 873         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 874                 .FLOWID_SEPARATOR + routerID;
 875     }
 876
 877     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 878         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 879                 .FLOWID_SEPARATOR + routerID;
 880     }
 881
 882     private String getGroupIdKey(String routerName) {
 883         return "snatmiss." + routerName;
 884     }
 885
 886     protected long createGroupId(String groupIdKey) {
 887         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 888             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 889             .build();
 890         try {
 891             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 892             RpcResult<AllocateIdOutput> rpcResult = result.get();
 893             return rpcResult.getResult().getIdValue();
 894         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 895             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 896         }
 897         return 0;
 898     }
 899
 900     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 901         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 902         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 903         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 904         List<BucketInfo> listBucketInfo = new ArrayList<>();
 905
 906         if (ifNamePrimary != null) {
 907             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 908             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 909                     interfaceManager, ifNamePrimary, routerId, true);
 910             if (listActionInfoPrimary.isEmpty()) {
 911                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 912                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 913                         ifNamePrimary);
 914             }
 915         } else {
 916             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 917                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 918         }
 919         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 920
 921         listBucketInfo.add(0, bucketPrimary);
 922         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 923     }
 924
 925     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 926                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 927         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 928         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 929         List<BucketInfo> listBucketInfo = new ArrayList<>();
 930
 931         if (ifNamePrimary != null) {
 932             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 933                     ifNamePrimary);
 934             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 935                     interfaceManager, ifNamePrimary, routerId, true);
 936             if (listActionInfoPrimary.isEmpty()) {
 937                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 938                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 939                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 940             }
 941         } else {
 942             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 943                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 944         }
 945         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 946
 947         listBucketInfo.add(0, bucketPrimary);
 948         return listBucketInfo;
 949     }
 950
 951     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 952         TypedWriteTransaction<Configuration> confTx) {
 953
 954        /*
 955         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 956         */
 957
 958         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 959
 960 /*
 961         List<BucketInfo> listBucketInfo = new ArrayList<>();
 962         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 963         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 964         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 965         listBucketInfo.add(0, bucketPrimary);
 966 */
 967
 968         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 969         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 970         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 971         installNaptPfibEntry(dpnId, routerId, confTx);
 972         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 973         if (networkId != null) {
 974             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 975             if (vpnUuid != null) {
 976                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 977                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 978                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 979                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 980                     if (vpnId != NatConstants.INVALID_ID) {
 981                         installNaptPfibEntry(dpnId, vpnId, null);
 982                     }
 983                     return Collections.emptyList();
 984                 });
 985             } else {
 986                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 987             }
 988         } else {
 989             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 990         }
 991     }
 992
 993     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 994         List<BucketInfo> listBucketInfo = new ArrayList<>();
 995         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 996         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 997         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 998         listBucketInfo.add(0, bucketPrimary);
 999         return listBucketInfo;
1000     }
1001
1002     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
1003             @Nullable TypedWriteTransaction<Configuration> confTx) {
1004         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1005         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1006         if (confTx != null) {
1007             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1008         } else {
1009             mdsalManager.installFlow(naptPfibFlowEntity);
1010         }
1011     }
1012
1013     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1014
1015         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1016         List<MatchInfo> matches = new ArrayList<>();
1017         matches.add(MatchEthernetType.IPV4);
1018         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1019
1020         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1021         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1022         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1023         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1024         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1025
1026         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1027         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1028             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1029             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1030         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1031         return flowEntity;
1032     }
1033
1034     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1035         long routerId, String routerName, String externalIp) {
1036         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1037             dpnId, routerId, externalIp);
1038         Uuid networkId = router.getNetworkId();
1039         if (networkId == null) {
1040             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1041             return;
1042         }
1043         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1044         if (vpnName == null) {
1045             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1046                     + "configuration {} in router {}", networkId, externalIp, routerId);
1047             return;
1048         }
1049         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1050             externalIp, networkId, router, confTx);
1051         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1052             dpnId, routerId, externalIp);
1053     }
1054
1055     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1056                                                 final long routerId, final String routerName, final String externalIp,
1057                                                 final Uuid extNetworkId, @Nullable final Routers router,
1058                                                 final TypedWriteTransaction<Configuration> confTx) {
1059         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1060                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1061         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1062         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1063         if (rd == null || rd.isEmpty()) {
1064             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1065             return;
1066         }
1067         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1068         if (extNwProvType == null) {
1069             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1070             return;
1071         }
1072         if (extNwProvType == ProviderTypes.VXLAN) {
1073             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1074                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1075             return;
1076         }
1077         //Generate VPN label for the external IP
1078         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1079             .setIpPrefix(externalIp).build();
1080         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1081
1082         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1083         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1084             if (result.isSuccessful()) {
1085                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1086                 GenerateVpnLabelOutput output = result.getResult();
1087                 final long label = output.getLabel();
1088
1089                 int externalIpInDsFlag = 0;
1090                 //Get IPMaps from the DB for the router ID
1091                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1092                 for (IpMap dbIpMap : dbIpMaps) {
1093                     String dbExternalIp = dbIpMap.getExternalIp();
1094                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1095                     if (dbExternalIp.contains(externalIp)) {
1096                         String dbInternalIp = dbIpMap.getInternalIp();
1097                         IpMapKey dbIpMapKey = dbIpMap.key();
1098                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1099                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1100                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1101                             .setExternalIp(dbExternalIp).setLabel(label).build();
1102                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1103                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1104                         externalIpInDsFlag++;
1105                     }
1106                 }
1107                 if (externalIpInDsFlag <= 0) {
1108                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1109                             + "Failed to update label {} for routerId {} in DS",
1110                             externalIp, label, routerId);
1111                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1112                         + " found in DS for router %s", label, externalIp, routerId);
1113                     return Futures.immediateFailedFuture(new Exception(errMsg));
1114                 }
1115                 //Inform BGP
1116                 long l3vni = 0;
1117                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1118                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1119                 }
1120                 Routers extRouter = router != null ? router :
1121                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1122                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1123                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1124                     RouteOrigin.STATIC, dpnId);
1125
1126                 //Install custom FIB routes
1127                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1128                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1129                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1130                         extNwProvType);
1131                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1132
1133                 //Install custom FIB routes - FIB table.
1134                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1135                         routerName, externalIp);
1136                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1137                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1138                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1139                 }
1140                 String externalVpn = vpnName;
1141                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1142                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1143                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1144                             externalSubnetId);
1145                     if (externalSubnet.isPresent()) {
1146                         externalVpn =  externalSubnetId.getValue();
1147                     }
1148                 }
1149                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1150                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1151                     .setVpnName(externalVpn)
1152                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1153                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1154                     .setInstruction(fibTableCustomInstructions).build();
1155                 return fibService.createFibEntry(input);
1156             } else {
1157                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1158                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1159                     externalIp, vpnName, result.getErrors());
1160                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1161             }
1162         }, MoreExecutors.directExecutor());
1163
1164         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1165
1166             @Override
1167             public void onFailure(@NonNull Throwable error) {
1168                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1169             }
1170
1171             @Override
1172             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1173                 if (result.isSuccessful()) {
1174                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1175                             externalIp);
1176                 } else {
1177                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1178                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1179                 }
1180             }
1181         }, MoreExecutors.directExecutor());
1182     }
1183
1184     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1185             String externalIp) {
1186         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1187         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1188         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1189                 externalIp, router);
1190         int instructionIndex = 0;
1191         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1192             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1193             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1194                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1195             instructionIndex++;
1196         }
1197
1198         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1199         return fibTableCustomInstructions;
1200     }
1201
1202     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1203         TypedWriteTransaction<Configuration> confTx) {
1204         List<MatchInfo> matches = new ArrayList<>();
1205         matches.add(MatchEthernetType.MPLS_UNICAST);
1206         matches.add(new MatchMplsLabel(serviceId));
1207
1208         List<Instruction> instructions = new ArrayList<>();
1209         List<ActionInfo> actionsInfos = new ArrayList<>();
1210         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1211         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1212         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1213         instructions.add(writeInstruction);
1214         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1215
1216         // Install the flow entry in L3_LFIB_TABLE
1217         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1218
1219         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1220             10, flowRef, 0, 0,
1221             COOKIE_VM_LFIB_TABLE, matches, instructions);
1222
1223         mdsalManager.addFlow(confTx, dpId, flowEntity);
1224
1225         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1226     }
1227
1228     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1229         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1230         ProviderTypes extNwProvType) {
1231
1232         List<MatchInfo> mkMatches = new ArrayList<>();
1233
1234         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1235             dpnId, serviceId, customInstructions);
1236
1237         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1238             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1239         } else {
1240             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1241         }
1242
1243         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1244             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1245             String.format("%s:%d", "TST Flow Entry ", serviceId),
1246             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1247
1248         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1249     }
1250
1251     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1252         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1253             new RouterIdsKey(routerId)).build();
1254     }
1255
1256     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1257         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1258                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1259     }
1260
1261     @Override
1262     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1263         String routerName = original.getRouterName();
1264         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1265         if (routerId == NatConstants.INVALID_ID) {
1266             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1267             return;
1268         }
1269         // Check if its update on SNAT flag
1270         boolean originalSNATEnabled = original.isEnableSnat();
1271         boolean updatedSNATEnabled = update.isEnableSnat();
1272         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1273             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1274         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1275          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1276          */
1277         long bgpVpnId = NatConstants.INVALID_ID;
1278         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1279         if (bgpVpnUuid != null) {
1280             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1281         }
1282         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1283         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1284             // Router has no interface attached
1285             return;
1286         }
1287         final long finalBgpVpnId = bgpVpnId;
1288         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1289             List<ListenableFuture<Void>> futures = new ArrayList<>();
1290             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1291                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1292                     Uuid networkId = original.getNetworkId();
1293                     if (originalSNATEnabled != updatedSNATEnabled) {
1294                         if (originalSNATEnabled) {
1295                             //SNAT disabled for the router
1296                             Uuid networkUuid = original.getNetworkId();
1297                             LOG.info("update : SNAT disabled for Router {}", routerName);
1298                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1299                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1300                                     removeFlowInvTx);
1301                         } else {
1302                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1303                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1304                                     true, writeFlowInvTx);
1305                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1306                         }
1307                     }
1308                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1309                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1310                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1311                     }
1312
1313                     //Check if the Update is on External IPs
1314                     LOG.debug("update : Checking if this is update on External IPs");
1315                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1316                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1317
1318                     //Check if the External IPs are added during the update.
1319                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1320                     addedExternalIps.removeAll(originalExternalIps);
1321                     if (addedExternalIps.size() != 0) {
1322                         LOG.debug("update : Start processing of the External IPs addition during the update "
1323                                 + "operation");
1324                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1325                                 update.getExtGwMacAddress(), dpnId,
1326                                 update.getNetworkId());
1327
1328                         for (String addedExternalIp : addedExternalIps) {
1329                 /*
1330                     1) Do nothing in the IntExtIp model.
1331                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1332                  */
1333                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1334                             String externalIp = externalIpParts[0];
1335                             String externalIpPrefix = externalIpParts[1];
1336                             String externalpStr = externalIp + "/" + externalIpPrefix;
1337                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1338                                             + "router ID {} in the ExternalIpsCounter model.",
1339                                     externalpStr, routerId);
1340                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1341                         }
1342                         LOG.debug(
1343                                 "update : End processing of the External IPs addition during the update operation");
1344                     }
1345
1346                     //Check if the External IPs are removed during the update.
1347                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1348                     removedExternalIps.removeAll(updatedExternalIps);
1349                     if (removedExternalIps.size() > 0) {
1350                         LOG.debug("update : Start processing of the External IPs removal during the update "
1351                                 + "operation");
1352                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1353                                 removedExternalIps, original.getExtGwMacAddress(),
1354                                 dpnId, networkId);
1355
1356                         for (String removedExternalIp : removedExternalIps) {
1357                 /*
1358                     1) Remove the mappings in the IntExt IP model which has external IP.
1359                     2) Remove the external IP in the ExternalCounter model.
1360                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1361                        least loaded external IP.
1362                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1363                     4) Increase the count of the allocated external IP by one.
1364                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1365                      i.e. the route for the external IP is absent.
1366                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1367                      the removed external IPs and also from the model.
1368                     7) Advertise to the BGP for removing the route for the removed external IPs.
1369                  */
1370
1371                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1372                             String externalIp = externalIpParts[0];
1373                             String externalIpPrefix = externalIpParts[1];
1374                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1375
1376                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1377                                     + "entries for removed external IP {}", externalIpAddrStr);
1378                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1379                             String vpnName = "";
1380                             if (vpnUuId != null) {
1381                                 vpnName = vpnUuId.getValue();
1382                             }
1383                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1384                                     update.getExtGwMacAddress(), removeFlowInvTx);
1385
1386                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1387                             //Get the internal IPs which are associated to the removed external IPs
1388                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1389                             List<String> removedInternalIps = new ArrayList<>();
1390                             for (IpMap ipMap : ipMaps) {
1391                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1392                                     removedInternalIps.add(ipMap.getInternalIp());
1393                                 }
1394                             }
1395
1396                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1397                             for (String removedInternalIp : removedInternalIps) {
1398                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1399                                                 + "router ID {} from the IntExtIP model",
1400                                         removedInternalIp, routerId);
1401                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1402                             }
1403
1404                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1405                                             + "router ID {} from the ExternalIpsCounter model.",
1406                                     externalIpAddrStr, routerId);
1407                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1408
1409                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1410                                     + "whose external IPs were removed.");
1411                             for (String removedInternalIp : removedInternalIps) {
1412                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1413                                         removedInternalIp, writeFlowInvTx);
1414                             }
1415                             LOG.debug("update : Remove the NAPT translation entries from "
1416                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1417                             //Get the internalIP and internal Port which were associated to the removed external IP.
1418                             List<Integer> externalPorts = new ArrayList<>();
1419                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1420                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1421                                     .builder(IntextIpPortMap.class)
1422                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1423                             Optional<IpPortMapping> ipPortMapping;
1424                             try {
1425                                 ipPortMapping = SingleTransactionDataBroker
1426                                             .syncReadOptional(dataBroker,
1427                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1428                             } catch (ReadFailedException e) {
1429                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1430                                 ipPortMapping = Optional.absent();
1431                             }
1432
1433                             if (ipPortMapping.isPresent()) {
1434                                 for (IntextIpProtocolType intextIpProtocolType :
1435                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1436                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1437                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1438                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1439                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1440                                             externalPorts.add(ipPortExternal.getPortNum());
1441                                             List<String> removedInternalIpPorts =
1442                                                     protoTypesIntIpPortsMap.get(protoType);
1443                                             if (removedInternalIpPorts != null) {
1444                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1445                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1446                                             } else {
1447                                                 removedInternalIpPorts = new ArrayList<>();
1448                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1449                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1450                                             }
1451                                         }
1452                                     }
1453                                 }
1454                             }
1455
1456                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1457                             // the removed external IP.
1458                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1459                                     protoTypesIntIpPortsMap.entrySet();
1460                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1461                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1462                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1463                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1464                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1465                                     // Remove the IP port map from the intext-ip-port-map model,
1466                                     // which were containing the removed external IP
1467                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1468                                             protocolType);
1469                                     //Remove the IP port incomint packer map.
1470                                     naptPacketInHandler.removeIncomingPacketMap(
1471                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1472                                     String[] removedInternalIpPortParts = removedInternalIpPort
1473                                             .split(NatConstants.COLON_SEPARATOR);
1474                                     if (removedInternalIpPortParts.length == 2) {
1475                                         String removedInternalIp = removedInternalIpPortParts[0];
1476                                         String removedInternalPort = removedInternalIpPortParts[1];
1477                                         List<String> removedInternalPortsList =
1478                                                 internalIpPortMap.get(removedInternalPort);
1479                                         if (removedInternalPortsList != null) {
1480                                             removedInternalPortsList.add(removedInternalPort);
1481                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1482                                         } else {
1483                                             removedInternalPortsList = new ArrayList<>();
1484                                             removedInternalPortsList.add(removedInternalPort);
1485                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1486                                         }
1487                                     }
1488                                 }
1489                             }
1490
1491                             // Delete the entry from SnatIntIpPortMap DS
1492                             Set<String> internalIps = internalIpPortMap.keySet();
1493                             for (String internalIp : internalIps) {
1494                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1495                                         + "model SnatIntIpPortMap", internalIp);
1496                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1497                             }
1498
1499                             naptManager.removeNaptPortPool(externalIp);
1500
1501                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1502                                     + "the removed external IP {}", externalIp);
1503                             for (Integer externalPort : externalPorts) {
1504                                 //Remove the NAPT translation entries from Inbound NAPT table
1505                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1506                                         routerId, externalIp, externalPort);
1507                             }
1508
1509                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1510                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1511                                 String internalIp = internalIpPort.getKey();
1512                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1513                                         + "for the removed internal IP {}", internalIp);
1514                                 List<String> internalPorts = internalIpPort.getValue();
1515                                 for (String internalPort : internalPorts) {
1516                                     //Remove the NAPT translation entries from Outbound NAPT table
1517                                     naptPacketInHandler.removeIncomingPacketMap(
1518                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1519                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1520                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1521                                             routerId, internalIp, Integer.parseInt(internalPort));
1522                                 }
1523                             }
1524                         }
1525                         LOG.debug(
1526                                 "update : End processing of the External IPs removal during the update operation");
1527                     }
1528
1529                     //Check if its Update on subnets
1530                     LOG.debug("update : Checking if this is update on subnets");
1531                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1532                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1533                     Set<Uuid> addedSubnetIds =
1534                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1535                     if (originalSubnetIds != null) {
1536                         addedSubnetIds.removeAll(originalSubnetIds);
1537                     }
1538
1539                     //Check if the Subnet IDs are added during the update.
1540                     if (addedSubnetIds.size() != 0) {
1541                         LOG.debug(
1542                                 "update : Start processing of the Subnet IDs addition during the update operation");
1543                         for (Uuid addedSubnetId : addedSubnetIds) {
1544                 /*
1545                     1) Select the least loaded external IP for the subnet and store the mapping of the
1546                     subnet IP and the external IP in the IntExtIp model.
1547                     2) Increase the count of the selected external IP by one.
1548                     3) Advertise to the BGP if external IP is allocated for the first time for the
1549                     router i.e. the route for the external IP is absent.
1550                  */
1551                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1552                             if (subnetIp != null) {
1553                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1554                                         writeFlowInvTx);
1555                             }
1556                         }
1557                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1558                     }
1559
1560                     //Check if the Subnet IDs are removed during the update.
1561                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1562                     removedSubnetIds.removeAll(updatedSubnetIds);
1563                     if (removedSubnetIds.size() != 0) {
1564                         LOG.debug(
1565                                 "update : Start processing of the Subnet IDs removal during the update operation");
1566                         for (Uuid removedSubnetId : removedSubnetIds) {
1567                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1568                             if (subnetAddr != null) {
1569                     /*
1570                         1) Remove the subnet IP and the external IP in the IntExtIp map
1571                         2) Decrease the count of the coresponding external IP by one.
1572                         3) Advertise to the BGP for removing the routes of the corresponding external
1573                         IP if its not allocated to any other internal IP.
1574                     */
1575
1576                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1577                                         subnetAddr[0] + "/" + subnetAddr[1]);
1578                                 if (externalIp == null) {
1579                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1580                                             routerId, subnetAddr[0]);
1581                                     return;
1582                                 }
1583
1584                                 naptManager.updateCounter(routerId, externalIp, false);
1585                                 // Traverse entire model of external-ip counter whether external ip is not
1586                                 // used by any other internal ip in any router
1587                                 if (!isExternalIpAllocated(externalIp)) {
1588                                     LOG.debug("update : external ip is not allocated to any other "
1589                                             + "internal IP so proceeding to remove routes");
1590                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1591                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1592                                             removeFlowInvTx);
1593                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1594                                                     + "router {} with networkId {} and externalIp {}",
1595                                             dpnId, routerId, networkId, externalIp);
1596                                 }
1597
1598                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1599                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1600                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1601                             }
1602                         }
1603                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1604                     }
1605                 }));
1606             }));
1607             return futures;
1608         }, NatConstants.NAT_DJC_MAX_RETRIES);
1609     }
1610
1611     private boolean isExternalIpAllocated(String externalIp) {
1612         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1613         Optional<ExternalIpsCounter> externalCountersData;
1614         try {
1615             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1616                         LogicalDatastoreType.OPERATIONAL, id);
1617         } catch (ReadFailedException e) {
1618             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1619             externalCountersData = Optional.absent();
1620         }
1621         if (externalCountersData.isPresent()) {
1622             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1623             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1624                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1625                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1626                         if (externalIpCount.getCounter() != 0) {
1627                             return true;
1628                         }
1629                         break;
1630                     }
1631                 }
1632             }
1633         }
1634         return false;
1635     }
1636
1637     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1638             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1639         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1640         try {
1641             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1642             if (address instanceof Inet6Address) {
1643                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1644                 return;
1645             }
1646         } catch (UnknownHostException e) {
1647             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1648             return;
1649         }
1650         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1651         if (leastLoadedExtIpAddr != null) {
1652             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1653             String leastLoadedExtIp = externalIpParts[0];
1654             String leastLoadedExtIpPrefix = externalIpParts[1];
1655             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1656             subnetIp = subnetIpParts[0];
1657             String subnetIpPrefix = subnetIpParts[1];
1658             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1659             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1660                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1661                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1662             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1663
1664
1665             // Check if external IP is already assigned a route. (i.e. External IP is previously
1666             // allocated to any of the subnets)
1667             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1668             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1669             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1670             if (label != null) {
1671                 //update
1672                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1673                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1674                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1675                     label, internalIp, leastLoadedExtIpAddrStr);
1676                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1677                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1678                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1679                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1680                 return;
1681             }
1682
1683             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1684             // for the first time and hence not having a route.
1685             //Get the VPN Name using the network ID
1686             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1687             if (vpnName != null) {
1688                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1689                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1690                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1691                         + "added after gateway-set");
1692                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1693                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1694                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1695                         return;
1696                     }
1697                 }
1698                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1699                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1700                     writeFlowInvTx);
1701             }
1702         }
1703     }
1704
1705     @Nullable
1706     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1707         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1708         for (IpMap ipMap : ipMaps) {
1709             if (ipMap.getExternalIp().equals(externalIp)) {
1710                 if (ipMap.getLabel() != null) {
1711                     return ipMap.getLabel();
1712                 }
1713             }
1714         }
1715         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1716         return null;
1717     }
1718
1719     @Override
1720     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1721         LOG.trace("remove : Router delete method");
1722         /*
1723         ROUTER DELETE SCENARIO
1724         1) Get the router ID from the event.
1725         2) Build the cookie information from the router ID.
1726         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1727         4) Build the flow with the cookie value.
1728         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1729         6) Remove the flows from the other switches which points to the primary and secondary
1730          switches for the flows related the router ID.
1731         7) Get the list of external IP address maintained for the router ID.
1732         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1733         9) Withdraw the corresponding routes from the BGP.
1734          */
1735
1736         if (identifier == null || router == null) {
1737             LOG.error("remove : returning without processing since routers is null");
1738             return;
1739         }
1740
1741         String routerName = router.getRouterName();
1742         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1743             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1744                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1745                         routerName);
1746                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1747                 if (routerId == NatConstants.INVALID_ID) {
1748                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1749                             routerName);
1750                     return;
1751                 }
1752                 long bgpVpnId = NatConstants.INVALID_ID;
1753                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1754                 if (bgpVpnUuid != null) {
1755                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1756                 }
1757                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1758                         tx);
1759                 Uuid networkUuid = router.getNetworkId();
1760
1761                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1762                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1763                     // No NAPT switch for external router, probably because the router is not attached to
1764                     // any
1765                     // internal networks
1766                     LOG.debug(
1767                             "No NAPT switch for router {}, check if router is attached to any internal "
1768                                     + "network",
1769                             routerName);
1770                     return;
1771                 } else {
1772                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1773                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1774                             routerId, tx);
1775                 }
1776                 natOverVxlanUtil.releaseVNI(routerName);
1777             })), NatConstants.NAT_DJC_MAX_RETRIES);
1778     }
1779
1780     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1781                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1782                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1783         LOG.info("handleDisableSnat : Entry");
1784         String routerName = router.getRouterName();
1785         try {
1786             if (routerFlag) {
1787                 removeNaptSwitch(routerName);
1788             } else {
1789                 updateNaptSwitch(routerName, BigInteger.ZERO);
1790             }
1791
1792             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1793             naptManager.removeExternalCounter(routerId);
1794
1795             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1796             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1797                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1798                     + "router ID {} from RouterNaptSwitch model", routerId);
1799                 return;
1800             }
1801             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1802                     networkUuid);
1803             if (extNwProvType == null) {
1804                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1805                 return;
1806             }
1807             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1808             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1809                     externalSubnetList, removeFlowInvTx, extNwProvType);
1810             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1811             String externalSubnetVpn = null;
1812             for (Uuid externalSubnetId : externalSubnetList) {
1813                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1814                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1815                 if (externalSubnet.isPresent()) {
1816                     externalSubnetVpn =  externalSubnetId.getValue();
1817                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1818                             router.getExtGwMacAddress(), removeFlowInvTx);
1819                 }
1820             }
1821             if (externalSubnetVpn == null) {
1822                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1823                         router.getExtGwMacAddress(), removeFlowInvTx);
1824             }
1825             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1826             // for the router ID.
1827             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1828                     + "router ID {} in the DS", routerId);
1829             naptManager.removeMapping(routerId);
1830         } catch (InterruptedException | ExecutionException e) {
1831             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1832         }
1833         LOG.info("handleDisableSnat : Exit");
1834     }
1835
1836     // TODO Clean up the exception handling
1837     @SuppressWarnings("checkstyle:IllegalCatch")
1838     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1839                                              @NonNull Collection<String> externalIps,
1840                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1841         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1842                 + "with internet vpn {}", routerName, vpnId);
1843         try {
1844             BigInteger naptSwitchDpnId = null;
1845             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1846                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1847             Optional<RouterToNaptSwitch> rtrToNapt;
1848             try {
1849                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1850                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1851             } catch (ReadFailedException e) {
1852                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1853                 rtrToNapt = Optional.absent();
1854             }
1855             if (rtrToNapt.isPresent()) {
1856                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1857             }
1858             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1859
1860             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1861                     writeFlowInvTx);
1862             try {
1863                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1864                 if (extGwMacAddress != null) {
1865                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1866                             + "External Router ID {}", extGwMacAddress, routerId);
1867                 } else {
1868                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1869                             + "External Router ID {}", routerId);
1870                     return;
1871                 }
1872                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1873                         writeFlowInvTx);
1874             } catch (Exception ex) {
1875                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1876                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1877             }
1878             natOverVxlanUtil.releaseVNI(vpnId);
1879         } catch (InterruptedException | ExecutionException e) {
1880             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1881                     + "with internet vpn {}", routerName, vpnId, e);
1882         }
1883         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1884                 routerName, vpnId);
1885     }
1886
1887     // TODO Clean up the exception handling
1888     @SuppressWarnings("checkstyle:IllegalCatch")
1889     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1890         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1891             .setPrimarySwitchId(naptSwitchId).build();
1892         try {
1893             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1894                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1895         } catch (Exception ex) {
1896             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1897                 naptSwitchId, routerName);
1898         }
1899         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1900             naptSwitchId, routerName);
1901     }
1902
1903     protected void removeNaptSwitch(String routerName) {
1904         // Remove router and switch from model
1905         InstanceIdentifier<RouterToNaptSwitch> id =
1906             InstanceIdentifier.builder(NaptSwitches.class)
1907                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1908         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1909         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1910         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1911         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1912     }
1913
1914     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1915                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1916                                                 @NonNull Collection<String> externalIps,
1917                                                 Collection<Uuid> externalSubnetList,
1918                                                 TypedReadWriteTransaction<Configuration> confTx,
1919                                                 ProviderTypes extNwProvType)
1920             throws InterruptedException, ExecutionException {
1921
1922         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1923         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1924
1925         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1926         // traffic which comes from the  VMs of the NAPT switches)
1927         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1928         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1929
1930         LOG.info(
1931             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1932                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1933         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1934
1935         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1936         // traffic which comes from the VMs of the non NAPT switches)
1937         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1938                 elanManager, idManager, routerId, routerName);
1939         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1940         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1941         LOG.info(
1942             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1943                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1944         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1945
1946         //Remove the flow table 25->44 from NAPT Switch
1947         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1948             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1949         }
1950
1951         //Remove the Outbound flow entry which forwards the packet to FIB Table
1952         LOG.info(
1953             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1954                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1955
1956         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1957             NwConstants.IP_PROT_TCP);
1958         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1959             outboundTcpNatFlowRef);
1960         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1961
1962         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1963             NwConstants.IP_PROT_UDP);
1964         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1965             outboundUdpNatFlowRef);
1966         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1967
1968         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1969             NwConstants.IP_PROT_ICMP);
1970         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1971             icmpDropFlowRef);
1972         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1973         boolean lastRouterOnExternalNetwork =
1974             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1975         if (lastRouterOnExternalNetwork) {
1976             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1977         }
1978         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1979         // External Subnet Vpn Id.
1980         for (Uuid externalSubnetId : externalSubnetList) {
1981             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1982             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1983                 dataBroker, externalSubnetId, routerName, dpnId)) {
1984                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1985                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1986                     natPfibSubnetFlowRef);
1987                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1988                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1989                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1990                     subnetVpnId, dpnId);
1991             }
1992         }
1993
1994         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1995         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1996         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1997
1998         LOG.info(
1999             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2000             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2001         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2002
2003         if (lastRouterOnExternalNetwork) {
2004             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2005             // - This does not work since ext-routers is deleted already - no network info
2006             //Get the VPN ID from the ExternalNetworks model
2007             long vpnId = -1;
2008             if (vpnName == null || vpnName.isEmpty()) {
2009                 // ie called from router delete cases
2010                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2011                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2012                 if (vpnUuid != null) {
2013                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2014                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2015                               + "or disableSNAT scenario", vpnId, networkId);
2016                 }
2017             } else {
2018                 // ie called from disassociate vpn case
2019                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2020                     vpnName);
2021                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2022                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2023                     vpnId);
2024             }
2025
2026             if (vpnId != NatConstants.INVALID_ID) {
2027                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2028                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2029                 FlowEntity natPfibVpnFlowEntity =
2030                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2031                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2032                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2033                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2034             }
2035         }
2036
2037         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2038         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2039         if (ipPortMapping == null) {
2040             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2041             return;
2042         }
2043
2044         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2045             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2046                 String ipPortInternal = ipPortMap.getIpPortInternal();
2047                 String[] ipPortParts = ipPortInternal.split(":");
2048                 if (ipPortParts.length != 2) {
2049                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2050                     return;
2051                 }
2052                 String internalIp = ipPortParts[0];
2053                 String internalPort = ipPortParts[1];
2054
2055                 //Build the flow for the outbound NAPT table
2056                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2057                     + NatConstants.COLON_SEPARATOR + internalPort);
2058                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2059                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2060                 FlowEntity outboundNaptFlowEntity =
2061                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2062
2063                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2064                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2065                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2066
2067                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2068                 String externalIp = ipPortExternal.getIpAddress();
2069                 int externalPort = ipPortExternal.getPortNum();
2070
2071                 //Build the flow for the inbound NAPT table
2072                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2073                     String.valueOf(routerId), externalIp, externalPort);
2074                 FlowEntity inboundNaptFlowEntity =
2075                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2076
2077                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2078                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2079                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2080             }
2081         }
2082     }
2083
2084     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2085                                                     @NonNull Collection<String> externalIps,
2086                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2087             throws ExecutionException, InterruptedException {
2088         long extVpnId = NatConstants.INVALID_ID;
2089         if (networkId != null) {
2090             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2091             if (vpnUuid != null) {
2092                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2093             } else {
2094                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2095             }
2096         } else {
2097             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2098             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2099         }
2100         if (extVpnId == NatConstants.INVALID_ID) {
2101             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2102             extVpnId = routerId;
2103         }
2104         for (String ip : externalIps) {
2105             String extIp = removeMaskFromIp(ip);
2106             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2107             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2108                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2109                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2110             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2111             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2112         }
2113     }
2114
2115     private String removeMaskFromIp(String ip) {
2116         if (ip != null && !ip.trim().isEmpty()) {
2117             return ip.split("/")[0];
2118         }
2119         return ip;
2120     }
2121
2122     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2123                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2124                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2125             throws ExecutionException, InterruptedException {
2126         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2127         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2128
2129         //Remove the NAPT PFIB TABLE entry
2130         long vpnId = -1;
2131         if (vpnName != null) {
2132             // ie called from disassociate vpn case
2133             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2134                     + "with vpnName {}", vpnName);
2135             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2136             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2137                     vpnId);
2138         }
2139
2140         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2141                 networkId, routerName, dpnId)) {
2142             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2143             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2144             FlowEntity natPfibVpnFlowEntity =
2145                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2146             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2147                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2148             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2149
2150             // Remove IP-PORT active NAPT entries and release port from IdManager
2151             // For the router ID get the internal IP , internal port and the corresponding
2152             // external IP and external Port.
2153             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2154             if (ipPortMapping == null) {
2155                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2156                 return;
2157             }
2158             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2159                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2160                     String ipPortInternal = ipPortMap.getIpPortInternal();
2161                     String[] ipPortParts = ipPortInternal.split(":");
2162                     if (ipPortParts.length != 2) {
2163                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2164                                 + "and port");
2165                         return;
2166                     }
2167                     String internalIp = ipPortParts[0];
2168                     String internalPort = ipPortParts[1];
2169
2170                     //Build the flow for the outbound NAPT table
2171                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2172                             + NatConstants.COLON_SEPARATOR + internalPort);
2173                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2174                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2175                     FlowEntity outboundNaptFlowEntity =
2176                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2177
2178                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2179                             + "active switch with the DPN ID {} and router ID {}",
2180                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2181                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2182
2183                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2184                     String externalIp = ipPortExternal.getIpAddress();
2185                     int externalPort = ipPortExternal.getPortNum();
2186
2187                     //Build the flow for the inbound NAPT table
2188                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2189                         String.valueOf(routerId), externalIp, externalPort);
2190                     FlowEntity inboundNaptFlowEntity =
2191                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2192
2193                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2194                             + "active active switch with the DPN ID {} and router ID {}",
2195                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2196                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2197
2198                     // Finally release port from idmanager
2199                     String internalIpPort = internalIp + ":" + internalPort;
2200                     naptManager.removePortFromPool(internalIpPort, externalIp);
2201
2202                     //Remove sessions from models
2203                     naptManager.removeIpPortMappingForRouterID(routerId);
2204                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2205                 }
2206             }
2207         } else {
2208             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2209         }
2210     }
2211
2212     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2213             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2214             throws ExecutionException, InterruptedException {
2215         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2216
2217         // Remove the flows from the other switches which points to the primary and secondary switches
2218         // for the flows related the router ID.
2219         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2220         if (allSwitchList.isEmpty()) {
2221             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2222             return;
2223         }
2224         for (BigInteger dpnId : allSwitchList) {
2225             if (!naptSwitchDpnId.equals(dpnId)) {
2226                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2227
2228                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2229                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2230                 FlowEntity preSnatFlowEntity =
2231                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2232
2233                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2234                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2235                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2236
2237                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2238                 long groupId = createGroupId(getGroupIdKey(routerName));
2239
2240                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2241                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2242                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2243             }
2244         }
2245     }
2246
2247     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2248                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2249                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2250             throws ExecutionException, InterruptedException {
2251         //Withdraw the corresponding routes from the BGP.
2252         //Get the network ID using the router ID.
2253         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2254                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2255         if (networkUuid == null) {
2256             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2257             return;
2258         }
2259
2260         if (externalIps.isEmpty()) {
2261             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2262             return;
2263         }
2264
2265         if (vpnName == null) {
2266             //Get the VPN Name using the network ID
2267             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2268             if (vpnName == null) {
2269                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2270                     networkUuid, routerId);
2271                 return;
2272             }
2273         }
2274         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2275
2276         //Remove custom FIB routes
2277         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2278         for (String extIp : externalIps) {
2279             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2280         }
2281     }
2282
2283     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2284                                             final Uuid networkUuid, String extGwMacAddress,
2285                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2286             throws ExecutionException, InterruptedException {
2287         clearBgpRoutes(extIp, vpnName);
2288         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2289                 removeFlowInvTx);
2290     }
2291
2292     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2293                                              String vpnName, Uuid extNetworkId, long tempLabel,
2294                                              String gwMacAddress, boolean switchOver,
2295                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2296             throws ExecutionException, InterruptedException {
2297         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2298         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2299         if (routerName == null) {
2300             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2301             return;
2302         }
2303         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2304         if (extNwProvType == null) {
2305             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2306             return;
2307         }
2308         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2309          * external network provided type is VxLAN
2310          */
2311         if (extNwProvType == ProviderTypes.VXLAN) {
2312             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2313             );
2314             return;
2315         }
2316         if (tempLabel < 0) {
2317             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2318                     extIp, routerId);
2319             return;
2320         }
2321         final long label = tempLabel;
2322         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2323         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2324             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2325             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2326             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2327                     extRouter);
2328
2329             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2330                     externalSubnetId);
2331
2332             if (externalSubnet.isPresent()) {
2333                 vpnName =  externalSubnetId.getValue();
2334             }
2335         }
2336         final String externalVpn = vpnName;
2337         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2338                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2339                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2340         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2341
2342         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2343         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2344         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2345             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2346             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2347         }
2348         if (!switchOver) {
2349             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2350                 Futures.transformAsync(future, result -> {
2351                     //Release label
2352                     if (result.isSuccessful()) {
2353                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2354                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2355                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2356                         return vpnService.removeVpnLabel(labelInput);
2357                     } else {
2358                         String errMsg =
2359                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2360                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2361                         LOG.error(errMsg);
2362                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2363                     }
2364                 }, MoreExecutors.directExecutor());
2365
2366             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2367
2368                 @Override
2369                 public void onFailure(@NonNull Throwable error) {
2370                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2371                         + "got external ip {}", label, extIp, error);
2372                 }
2373
2374                 @Override
2375                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2376                     if (result.isSuccessful()) {
2377                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2378                             + "from VPN {}", externalIp, externalVpn);
2379                     } else {
2380                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2381                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2382                     }
2383                 }
2384             }, MoreExecutors.directExecutor());
2385         } else {
2386             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2387                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2388         }
2389     }
2390
2391     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2392                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2393                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2394             throws ExecutionException, InterruptedException {
2395         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2396         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2397         if (routerName == null) {
2398             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2399             return;
2400         }
2401         //Get the external network provider type from networkId
2402         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2403         if (extNwProvType == null) {
2404             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2405                     networkUuid);
2406             return;
2407         }
2408         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2409          *  external network provided type is VxLAN
2410          */
2411         if (extNwProvType == ProviderTypes.VXLAN) {
2412             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2413             return;
2414         }
2415         //Get IPMaps from the DB for the router ID
2416         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2417         if (dbIpMaps.isEmpty()) {
2418             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2419             return;
2420         }
2421
2422         long tempLabel = NatConstants.INVALID_ID;
2423         for (IpMap dbIpMap : dbIpMaps) {
2424             String dbExternalIp = dbIpMap.getExternalIp();
2425             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2426             //Select the IPMap, whose external IP is the IP for which FIB is installed
2427             if (extIp.equals(dbExternalIp)) {
2428                 tempLabel = dbIpMap.getLabel();
2429                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2430                     tempLabel, dbExternalIp, routerId);
2431                 break;
2432             }
2433         }
2434         if (tempLabel == NatConstants.INVALID_ID) {
2435             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2436                     extIp, routerId);
2437             return;
2438         }
2439
2440         final long label = tempLabel;
2441         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2442         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2443                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2444                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2445         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2446
2447         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2448         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2449         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2450             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2451             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2452         }
2453         if (!switchOver) {
2454             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2455                     Futures.transformAsync(future, result -> {
2456                         //Release label
2457                         if (result.isSuccessful()) {
2458                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2459                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2460                             return vpnService.removeVpnLabel(labelInput);
2461                         } else {
2462                             String errMsg =
2463                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2464                                             + "prefix %s Failed - %s",
2465                                             dpnId, externalIp, result.getErrors());
2466                             LOG.error(errMsg);
2467                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2468                         }
2469                     }, MoreExecutors.directExecutor());
2470
2471             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2472
2473                 @Override
2474                 public void onFailure(@NonNull Throwable error) {
2475                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2476                 }
2477
2478                 @Override
2479                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2480                     if (result.isSuccessful()) {
2481                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2482                             + "from VPN {}", externalIp, vpnName);
2483                     } else {
2484                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2485                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2486                     }
2487                 }
2488             }, MoreExecutors.directExecutor());
2489         } else {
2490             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2491                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2492         }
2493     }
2494
2495     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2496             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2497             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2498         //Withdraw the corresponding routes from the BGP.
2499         //Get the network ID using the router ID.
2500         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2501                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2502         if (networkUuid == null) {
2503             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2504             return;
2505         }
2506
2507         if (externalIps == null || externalIps.isEmpty()) {
2508             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2509             return;
2510         }
2511
2512         if (vpnName == null) {
2513             //Get the VPN Name using the network ID
2514             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2515             if (vpnName == null) {
2516                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2517                     networkUuid, routerId);
2518                 return;
2519             }
2520         }
2521         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2522
2523         //Remove custom FIB routes
2524         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2525         for (String extIp : externalIps) {
2526             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2527                     writeFlowInvTx);
2528         }
2529     }
2530
2531     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2532         //Inform BGP about the route removal
2533         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2534         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2535         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2536     }
2537
2538     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2539             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2540         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2541         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2542             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2543         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2544     }
2545
2546     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2547             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2548         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2549         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2550         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2551         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2552     }
2553
2554     /**
2555      * router association to vpn.
2556      *
2557      * @param routerName - Name of router
2558      * @param routerId - router id
2559      * @param bgpVpnName BGP VPN name
2560      */
2561     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2562             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2563         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2564         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2565             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2566
2567             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2568
2569             if (bgpVpnId != NatConstants.INVALID_ID) {
2570                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2571                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2572                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2573                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2574                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2575                     getRoutersIdentifier(bgpVpnId), rtrs);
2576
2577                 // Get the allocated Primary NAPT Switch for this router
2578                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2579
2580                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2581                         routerId, bgpVpnId);
2582                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2583
2584                 // Get the group ID
2585                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2586                 createGroupId(getGroupIdKey(routerName));
2587                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2588                         extNwProvType);
2589             }
2590         }
2591     }
2592
2593     /**
2594      * router disassociation from vpn.
2595      *
2596      * @param routerName - Name of router
2597      * @param routerId - router id
2598      * @param bgpVpnName BGP VPN name
2599      */
2600     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2601             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2602         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2603         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2604             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2605             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2606
2607             // Get the allocated Primary NAPT Switch for this router
2608             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2609
2610             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2611             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2612
2613             // Get the group ID
2614             createGroupId(getGroupIdKey(routerName));
2615             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2616             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2617                     writeFlowInvTx, extNwProvType);
2618         }
2619     }
2620
2621     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2622         InstanceIdentifier<Routers> routerInstanceIndentifier =
2623             InstanceIdentifier.builder(ExtRouters.class)
2624                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2625         try {
2626             Optional<Routers> routerData = SingleTransactionDataBroker
2627                     .syncReadOptional(dataBroker,
2628                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2629             return routerData.isPresent() && routerData.get().isEnableSnat();
2630         } catch (ReadFailedException e) {
2631             LOG.error("Failed to read data for router id {}", routerUuid, e);
2632             return false;
2633         }
2634     }
2635
2636     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2637         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2638
2639         long changedVpnId = bgpVpnId;
2640         String idType = "BGP VPN";
2641         if (bgpVpnId == NatConstants.INVALID_ID) {
2642             changedVpnId = routerId;
2643             idType = "router";
2644         }
2645
2646         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2647         if (switches.isEmpty()) {
2648             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2649             return;
2650         }
2651         for (BigInteger dpnId : switches) {
2652             // Update the BGP VPN ID in the SNAT miss entry to group
2653             if (!dpnId.equals(primarySwitchId)) {
2654                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2655                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2656                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2657                 long groupId = createGroupId(getGroupIdKey(routerName));
2658                 if (!isSnatCfgd) {
2659                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2660                 }
2661
2662                 LOG.debug(
2663                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2664                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2665                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2666                 mdsalManager.addFlow(confTx, flowEntity);
2667             } else {
2668                 LOG.debug(
2669                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2670                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2671                 FlowEntity flowEntity =
2672                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2673                 mdsalManager.addFlow(confTx, flowEntity);
2674
2675                 LOG.debug(
2676                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2677                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2678                         idType, changedVpnId, primarySwitchId);
2679                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2680                         changedVpnId, confTx, extNwProvType);
2681
2682                 LOG.debug(
2683                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2684                                 + "which punts the packet to the controller in the Primary switch {}",
2685                         idType, changedVpnId, primarySwitchId);
2686                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2687
2688                 LOG.debug(
2689                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2690                                 + " outgoing packet to FIB Table in the Primary switch {}",
2691                         idType, changedVpnId, primarySwitchId);
2692                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2693
2694                 LOG.debug(
2695                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2696                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2697                                 + " {}", idType, changedVpnId, primarySwitchId);
2698                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2699
2700                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2701                         primarySwitchId);
2702                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2703                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2704                 if (vpnId != NatConstants.INVALID_ID) {
2705                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2706                 }
2707             }
2708         }
2709     }
2710
2711     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2712         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2713         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2714         if (ipPortMapping == null) {
2715             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2716             return;
2717         }
2718         // Get the External Gateway MAC Address
2719         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2720         if (extGwMacAddress != null) {
2721             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2722                     extGwMacAddress, routerId);
2723         } else {
2724             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2725                     routerId);
2726             return;
2727         }
2728         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2729             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2730                 String ipPortInternal = ipPortMap.getIpPortInternal();
2731                 String[] ipPortParts = ipPortInternal.split(":");
2732                 if (ipPortParts.length != 2) {
2733                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2734                     return;
2735                 }
2736                 String internalIp = ipPortParts[0];
2737                 String internalPort = ipPortParts[1];
2738                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2739                         internalIp, internalPort);
2740                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2741                 NAPTEntryEvent.Protocol protocol;
2742                 switch (protocolTypes) {
2743                     case TCP:
2744                         protocol = NAPTEntryEvent.Protocol.TCP;
2745                         break;
2746                     case UDP:
2747                         protocol = NAPTEntryEvent.Protocol.UDP;
2748                         break;
2749                     default:
2750                         protocol = NAPTEntryEvent.Protocol.TCP;
2751                 }
2752                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2753                 SessionAddress externalAddress =
2754                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2755                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2756                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2757                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2758                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2759                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2760             }
2761         }
2762     }
2763
2764     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2765                                                           long changedVpnId) {
2766
2767         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2768             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2769         List<MatchInfo> matches = new ArrayList<>();
2770         matches.add(MatchEthernetType.IPV4);
2771         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2772
2773         List<ActionInfo> actionsInfo = new ArrayList<>();
2774         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2775                 elanManager, idManager, changedVpnId, routerName);
2776         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2777         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2778                 actionsInfo);
2779         actionsInfo.add(new ActionGroup(groupId));
2780         List<InstructionInfo> instructions = new ArrayList<>();
2781         instructions.add(new InstructionApplyActions(actionsInfo));
2782         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2783         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2784             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2785             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2786
2787         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2788         return flowEntity;
2789     }
2790
2791     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2792                                                                         long changedVpnId) {
2793
2794         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2795                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2796         List<MatchInfo> matches = new ArrayList<>();
2797         matches.add(MatchEthernetType.IPV4);
2798         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2799
2800         List<InstructionInfo> instructions = new ArrayList<>();
2801         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2802
2803         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2804         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2805             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2806             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2807
2808         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2809         return flowEntity;
2810     }
2811
2812     // TODO : Replace this with ITM Rpc once its available with full functionality
2813     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2814         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2815
2816         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2817             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2818         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2819                 extNwProvType);
2820         mdsalManager.addFlow(confTx, flowEntity);
2821     }
2822
2823     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2824             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2825         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2826             dpId, routerName, changedVpnId);
2827         List<MatchInfo> matches = new ArrayList<>();
2828         matches.add(MatchEthernetType.IPV4);
2829
2830         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2831         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2832             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2833         }
2834         matches.add(new MatchTunnelId(tunnelId));
2835
2836         List<InstructionInfo> instructions = new ArrayList<>();
2837         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2838             MetaDataUtil.METADATA_MASK_VRFID));
2839         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2840         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2841         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2842         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2843             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2844             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2845         return flowEntity;
2846     }
2847
2848     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2849                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2850         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2851             dpnId, routerId, changedVpnId);
2852         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2853                 NwConstants.IP_PROT_TCP);
2854         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2855         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2856
2857         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2858                 NwConstants.IP_PROT_UDP);
2859         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2860         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2861
2862         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2863         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2864         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2865     }
2866
2867     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2868                                                            long changedVpnId, int protocol) {
2869         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2870             dpId, routerId, changedVpnId);
2871         BigInteger cookie = getCookieOutboundFlow(routerId);
2872         List<MatchInfo> matches = new ArrayList<>();
2873         matches.add(MatchEthernetType.IPV4);
2874         matches.add(new MatchIpProtocol((short)protocol));
2875         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2876
2877         List<InstructionInfo> instructions = new ArrayList<>();
2878         List<ActionInfo> actionsInfos = new ArrayList<>();
2879         actionsInfos.add(new ActionPuntToController());
2880         if (snatPuntTimeout != 0) {
2881             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2882         }
2883         instructions.add(new InstructionApplyActions(actionsInfos));
2884
2885         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2886         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2887             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2888         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2889         return flowEntity;
2890     }
2891
2892     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2893                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2894         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2895             dpnId, segmentId, changedVpnId);
2896         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2897         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2898     }
2899
2900     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2901
2902         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2903             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2904         List<MatchInfo> matches = new ArrayList<>();
2905         matches.add(MatchEthernetType.IPV4);
2906         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2907
2908         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2909         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2910         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2911         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2912         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2913
2914         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2915         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2916             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2917             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2918         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2919         return flowEntity;
2920     }
2921
2922     @Override
2923     protected ExternalRoutersListener getDataTreeChangeListener() {
2924         return ExternalRoutersListener.this;
2925     }
2926
2927     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2928                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2929         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2930                 routerName);
2931         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2932             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2933             if (subnetVpnId != -1) {
2934                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2935                     + "and vpnId {}", dpnId, subnetVpnId);
2936                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2937             }
2938         }
2939     }
2940 }