natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.Nonnull;
  35 import javax.annotation.Nullable;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.infra.Datastore.Configuration;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  49 import org.opendaylight.genius.mdsalutil.ActionInfo;
  50 import org.opendaylight.genius.mdsalutil.BucketInfo;
  51 import org.opendaylight.genius.mdsalutil.FlowEntity;
  52 import org.opendaylight.genius.mdsalutil.GroupEntity;
  53 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  54 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  55 import org.opendaylight.genius.mdsalutil.MatchInfo;
  56 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  69 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  72 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  73 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  78 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  79 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  80 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  81 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  82 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  83 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  84 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  85 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  86 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 150 import org.opendaylight.yangtools.yang.binding.DataObject;
 151 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 152 import org.opendaylight.yangtools.yang.common.RpcResult;
 153 import org.slf4j.Logger;
 154 import org.slf4j.LoggerFactory;
 155
 156 @Singleton
 157 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 158     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 159
 160     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 161     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 162
 163     private final DataBroker dataBroker;
 164     private final ManagedNewTransactionRunner txRunner;
 165     private final IMdsalApiManager mdsalManager;
 166     private final ItmRpcService itmManager;
 167     private final OdlInterfaceRpcService odlInterfaceRpcService;
 168     private final IdManagerService idManager;
 169     private final NaptManager naptManager;
 170     private final NAPTSwitchSelector naptSwitchSelector;
 171     private final IBgpManager bgpManager;
 172     private final VpnRpcService vpnService;
 173     private final FibRpcService fibService;
 174     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 175     private final NaptEventHandler naptEventHandler;
 176     private final NaptPacketInHandler naptPacketInHandler;
 177     private final IFibManager fibManager;
 178     private final IVpnManager vpnManager;
 179     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 180     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 181     private final NatMode natMode;
 182     private final INeutronVpnManager nvpnManager;
 183     private final IElanService elanManager;
 184     private final JobCoordinator coordinator;
 185     private final UpgradeState upgradeState;
 186     private final IInterfaceManager interfaceManager;
 187     private final int snatPuntTimeout;
 188
 189     @Inject
 190     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 191                                    final ItmRpcService itmManager,
 192                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 193                                    final IdManagerService idManager,
 194                                    final NaptManager naptManager,
 195                                    final NAPTSwitchSelector naptSwitchSelector,
 196                                    final IBgpManager bgpManager,
 197                                    final VpnRpcService vpnService,
 198                                    final FibRpcService fibService,
 199                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 200                                    final NaptEventHandler naptEventHandler,
 201                                    final NaptPacketInHandler naptPacketInHandler,
 202                                    final IFibManager fibManager,
 203                                    final IVpnManager vpnManager,
 204                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 205                                    final INeutronVpnManager nvpnManager,
 206                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 207                                    final NatserviceConfig config,
 208                                    final IElanService elanManager,
 209                                    final JobCoordinator coordinator,
 210                                    final UpgradeState upgradeState,
 211                                    final IInterfaceManager interfaceManager) {
 212         super(Routers.class, ExternalRoutersListener.class);
 213         this.dataBroker = dataBroker;
 214         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 215         this.mdsalManager = mdsalManager;
 216         this.itmManager = itmManager;
 217         this.odlInterfaceRpcService = odlInterfaceRpcService;
 218         this.idManager = idManager;
 219         this.naptManager = naptManager;
 220         this.naptSwitchSelector = naptSwitchSelector;
 221         this.bgpManager = bgpManager;
 222         this.vpnService = vpnService;
 223         this.fibService = fibService;
 224         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 225         this.naptEventHandler = naptEventHandler;
 226         this.naptPacketInHandler = naptPacketInHandler;
 227         this.fibManager = fibManager;
 228         this.vpnManager = vpnManager;
 229         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 230         this.nvpnManager = nvpnManager;
 231         this.elanManager = elanManager;
 232         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 233         this.coordinator = coordinator;
 234         this.upgradeState = upgradeState;
 235         this.interfaceManager = interfaceManager;
 236         if (config != null) {
 237             this.natMode = config.getNatMode();
 238             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 239         } else {
 240             this.natMode = NatMode.Controller;
 241             this.snatPuntTimeout = 0;
 242         }
 243     }
 244
 245     @Override
 246     @PostConstruct
 247     public void init() {
 248         LOG.info("{} init", getClass().getSimpleName());
 249         // This class handles ExternalRouters for Controller SNAT mode.
 250         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 251         if (natMode == NatMode.Controller) {
 252             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 253             NatUtil.createGroupIdPool(idManager);
 254         }
 255     }
 256
 257     @Override
 258     protected InstanceIdentifier<Routers> getWildCardPath() {
 259         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 260     }
 261
 262     @Override
 263     // TODO Clean up the exception handling
 264     @SuppressWarnings("checkstyle:IllegalCatch")
 265     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 266         // Populate the router-id-name container
 267         String routerName = routers.getRouterName();
 268         LOG.info("add : external router event for {}", routerName);
 269         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 270         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 271         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 272         try {
 273             if (routers.isEnableSnat()) {
 274                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 275                     () -> Collections.singletonList(
 276                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 277                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 278                             long bgpVpnId = NatConstants.INVALID_ID;
 279                             if (bgpVpnUuid != null) {
 280                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 281                             }
 282                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 283                             // Allocate Primary Napt Switch for this router
 284                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 285                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 286                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 287                             }
 288                         }
 289                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 290             } else {
 291                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 292             }
 293         } catch (Exception ex) {
 294             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 295                     routerName, ex);
 296         }
 297     }
 298
 299     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 300                                  TypedWriteTransaction<Configuration> confTx) {
 301         String routerName = routers.getRouterName();
 302         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 303
 304         naptManager.initialiseExternalCounter(routers, routerId);
 305         subnetRegisterMapping(routers, routerId);
 306
 307         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 308             primarySwitchId, routerName);
 309
 310         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 311                 routers.getNetworkId());
 312         if (extNwProvType == null) {
 313             LOG.error("handleEnableSnat : External Network Provider Type missing");
 314             return;
 315         }
 316
 317         if (bgpVpnId != NatConstants.INVALID_ID) {
 318             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 319                     extNwProvType);
 320         } else {
 321             // write metadata and punt
 322             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 323             // Now install entries in SNAT tables to point to Primary for each router
 324             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 325             for (BigInteger dpnId : switches) {
 326                 // Handle switches and NAPT switches separately
 327                 if (!dpnId.equals(primarySwitchId)) {
 328                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 329                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 330                 } else {
 331                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 332                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 333                 }
 334             }
 335         }
 336
 337         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 338         if (externalIps.isEmpty()) {
 339             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 340             return;
 341         } else {
 342             for (String externalIpAddrPrefix : externalIps) {
 343                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 344                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 345                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 346                 );
 347             }
 348         }
 349         LOG.debug("handleEnableSnat : Exit");
 350     }
 351
 352     private BigInteger getPrimaryNaptSwitch(String routerName) {
 353         // Allocate Primary Napt Switch for this router
 354         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 355         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 356             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 357                 primarySwitchId, routerName);
 358             return primarySwitchId;
 359         }
 360         // Validating and creating VNI pool during when NAPT switch is selected.
 361         // With Assumption this might be the first NAT service comes up.
 362         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 363                 NatConstants.ODL_VNI_POOL_NAME);
 364         // Allocated an id from VNI pool for the Router.
 365         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 366         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 367         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 368
 369         return primarySwitchId;
 370     }
 371
 372     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 373                                                      TypedWriteTransaction<Configuration> confTx) {
 374         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 375         if (extVpnId == NatConstants.INVALID_ID) {
 376             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 377             extVpnId = routerId;
 378         }
 379         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 380         if (externalIps.isEmpty()) {
 381             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 382                 dpnId, extVpnId);
 383             return;
 384         }
 385         for (String ip : externalIps) {
 386             Uuid subnetId = getSubnetIdForFixedIp(ip);
 387             if (subnetId != null) {
 388                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 389                 if (subnetVpnId != NatConstants.INVALID_ID) {
 390                     extVpnId = subnetVpnId;
 391                 }
 392                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 393                     dpnId, extVpnId, subnetId);
 394                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 395                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 396             }
 397         }
 398     }
 399
 400     @Nullable
 401     private Uuid getSubnetIdForFixedIp(String ip) {
 402         if (ip != null) {
 403             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 404             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 405             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 406         }
 407         LOG.error("getSubnetIdForFixedIp : ip is null");
 408         return null;
 409     }
 410
 411     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 412         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 413         List<Uuid> subnetList = routerEntry.getSubnetIds();
 414         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 415         int counter = 0;
 416         int extIpCounter = externalIps.size();
 417         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 418                 counter, extIpCounter);
 419         for (Uuid subnet : subnetList) {
 420             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 421             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 422                 .builder(Subnetmaps.class)
 423                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 424                 .build();
 425             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 426             if (sn.isPresent()) {
 427                 // subnets
 428                 Subnetmap subnetmapEntry = sn.get();
 429                 String subnetString = subnetmapEntry.getSubnetIp();
 430                 String[] subnetSplit = subnetString.split("/");
 431                 String subnetIp = subnetSplit[0];
 432                 try {
 433                     InetAddress address = InetAddress.getByName(subnetIp);
 434                     if (address instanceof Inet6Address) {
 435                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 436                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 437                         continue;
 438                     }
 439                 } catch (UnknownHostException e) {
 440                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 441                     return;
 442                 }
 443                 String subnetPrefix = "0";
 444                 if (subnetSplit.length == 2) {
 445                     subnetPrefix = subnetSplit[1];
 446                 }
 447                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 448                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 449                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 450                 //externalIps
 451                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 452                         counter, extIpCounter);
 453                 if (extIpCounter != 0) {
 454                     if (counter < extIpCounter) {
 455                         String[] ipSplit = externalIps.get(counter).split("/");
 456                         String externalIp = ipSplit[0];
 457                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 458                         if (ipSplit.length == 2) {
 459                             extPrefix = ipSplit[1];
 460                         }
 461                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 462                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 463                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 464                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 465                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 466                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 467                     } else {
 468                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 469                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 470                         String[] ipSplit = externalIps.get(counter).split("/");
 471                         String externalIp = ipSplit[0];
 472                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 473                         if (ipSplit.length == 2) {
 474                             extPrefix = ipSplit[1];
 475                         }
 476                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 477                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 478                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 479                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 480                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 481                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 482                             externalIp, extPrefix);
 483                     }
 484                 }
 485                 counter++;
 486                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 487             } else {
 488                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 489             }
 490         }
 491     }
 492
 493     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 494             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 495             throws ExecutionException, InterruptedException {
 496         //Check if BGP VPN exists. If exists then invoke the new method.
 497         if (bgpVpnId != NatConstants.INVALID_ID) {
 498             if (bgpVpnUuid != null) {
 499                 String bgpVpnName = bgpVpnUuid.getValue();
 500                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 501                     bgpVpnId, bgpVpnName);
 502                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 503                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 504                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 505             }
 506             if (create) {
 507                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 508             } else {
 509                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 510             }
 511             return;
 512         }
 513
 514         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 515         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 516     }
 517
 518     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 519             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 520         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 521         if (switches.isEmpty()) {
 522             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 523             return;
 524         }
 525         if (routerId == NatConstants.INVALID_ID) {
 526             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 527                     + "default NAT route in FIB", routerName);
 528             return;
 529         }
 530         for (BigInteger dpnId : switches) {
 531             if (create) {
 532                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 533                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 534                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 535             } else {
 536                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 537                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 538                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 539             }
 540         }
 541     }
 542
 543     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 544         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 545         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 546         if (dpnIds.isEmpty()) {
 547             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 548                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 549             return;
 550         }
 551         for (BigInteger dpnId : dpnIds) {
 552             if (bgpVpnId != NatConstants.INVALID_ID) {
 553                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 554                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 555                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 556             } else {
 557                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 558                     + "in dpn {} for the internal vpn", routerId, dpnId);
 559                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 560             }
 561         }
 562     }
 563
 564     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 565             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 566             throws ExecutionException, InterruptedException {
 567         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 568         if (dpnIds.isEmpty()) {
 569             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 570                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 571             return;
 572         }
 573         for (BigInteger dpnId : dpnIds) {
 574             if (bgpVpnId != NatConstants.INVALID_ID) {
 575                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 576                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 577                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 578             } else {
 579                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 580                     + "in dpn {} for the internal vpn", routerId, dpnId);
 581                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 582             }
 583         }
 584     }
 585
 586     // TODO Clean up the exception handling
 587     @SuppressWarnings("checkstyle:IllegalCatch")
 588     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 589                                                           InstanceIdentifier<T> path) {
 590         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 591
 592         try {
 593             return tx.read(datastoreType, path).get();
 594         } catch (Exception e) {
 595             throw new RuntimeException(e);
 596         }
 597     }
 598
 599     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 600         TypedWriteTransaction<Configuration> confTx) {
 601         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 602         if (routerId != NatConstants.INVALID_ID) {
 603             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 604                     primarySwitchId, routerId);
 605             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 606         } else {
 607             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 608                 + "createAndInstallMissEntry", routerName);
 609         }
 610     }
 611
 612     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 613         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 614                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 615     }
 616
 617     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 618         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 619                 .FLOWID_SEPARATOR + vpnId;
 620     }
 621
 622     public BigInteger getCookieOutboundFlow(long routerId) {
 623         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 624             BigInteger.valueOf(routerId));
 625     }
 626
 627     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 628         long l4SrcPortField;
 629         long l4DstPortField;
 630         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 631
 632         if (protocol == NwConstants.IP_PROT_TCP) {
 633             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 634             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 635         } else {
 636             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 637             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 638         }
 639         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 640                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 641                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 642                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 643                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 644                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 645                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 646                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 647                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 648                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 649                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 650                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 651                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 652                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 653                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 654
 655         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 656                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 657     }
 658
 659     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 660         List<MatchInfo> matches = new ArrayList<>();
 661         matches.add(MatchEthernetType.IPV4);
 662         matches.add(MatchIpProtocol.ICMP);
 663         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 664
 665         List<ActionInfo> actionInfos = new ArrayList<>();
 666         actionInfos.add(new ActionDrop());
 667
 668         List<InstructionInfo> instructions = new ArrayList<>();
 669         instructions.add(new InstructionApplyActions(actionInfos));
 670
 671         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 672                 NwConstants.IP_PROT_ICMP);
 673         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 674                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 675                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 676         return flow;
 677     }
 678
 679     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 680         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 681         BigInteger cookie = getCookieOutboundFlow(routerId);
 682         List<MatchInfo> matches = new ArrayList<>();
 683         matches.add(MatchEthernetType.IPV4);
 684         matches.add(new MatchIpProtocol((short)protocol));
 685         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 686
 687         List<InstructionInfo> instructions = new ArrayList<>();
 688         List<ActionInfo> actionsInfos = new ArrayList<>();
 689         actionsInfos.add(new ActionPuntToController());
 690         if (snatPuntTimeout != 0) {
 691             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 692         }
 693         instructions.add(new InstructionApplyActions(actionsInfos));
 694
 695         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 696         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 697             5, flowRef, 0, 0,
 698             cookie, matches, instructions);
 699         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 700         return flowEntity;
 701     }
 702
 703     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 704         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 705         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 706         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 707         mdsalManager.addFlow(confTx, tcpflowEntity);
 708
 709         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 710         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 711         mdsalManager.addFlow(confTx, udpflowEntity);
 712
 713         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 714         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 715         mdsalManager.addFlow(confTx, icmpDropFlow);
 716     }
 717
 718     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 719         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 720         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 721         try {
 722             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 723                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 724                     .setSourceDpid(srcDpId)
 725                     .setDestinationDpid(dstDpId)
 726                     .setTunnelType(tunType)
 727                     .build());
 728             rpcResult = result.get();
 729             if (!rpcResult.isSuccessful()) {
 730                 tunType = TunnelTypeGre.class;
 731                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 732                     .setSourceDpid(srcDpId)
 733                     .setDestinationDpid(dstDpId)
 734                     .setTunnelType(tunType)
 735                     .build());
 736                 rpcResult = result.get();
 737                 if (!rpcResult.isSuccessful()) {
 738                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 739                             rpcResult.getErrors());
 740                 } else {
 741                     return rpcResult.getResult().getInterfaceName();
 742                 }
 743                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 744                         rpcResult.getErrors());
 745             } else {
 746                 return rpcResult.getResult().getInterfaceName();
 747             }
 748         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 749             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 750                     + "between {} and {}", srcDpId, dstDpId, e);
 751         }
 752
 753         return null;
 754     }
 755
 756     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 757         TypedWriteTransaction<Configuration> confTx) {
 758
 759         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 760         // Install miss entry pointing to group
 761         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 762         mdsalManager.addFlow(confTx, flowEntity);
 763     }
 764
 765     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 766             String routerName, long routerId) {
 767         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 768             dpnId, bucketInfo.get(0));
 769         // Install the select group
 770         long groupId = createGroupId(getGroupIdKey(routerName));
 771         GroupEntity groupEntity =
 772             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 773         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 774         mdsalManager.syncInstallGroup(groupEntity);
 775         // Install miss entry pointing to group
 776         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 777         if (flowEntity == null) {
 778             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 779                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 780                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 781             return;
 782         }
 783         mdsalManager.installFlow(flowEntity);
 784     }
 785
 786     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 787         long groupId = createGroupId(getGroupIdKey(routerName));
 788         GroupEntity groupEntity =
 789             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 790         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 791         mdsalManager.syncInstallGroup(groupEntity);
 792         return groupId;
 793     }
 794
 795     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 796         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 797                 dpId, routerName, groupId);
 798         List<MatchInfo> matches = new ArrayList<>();
 799         matches.add(MatchEthernetType.IPV4);
 800         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 801
 802         List<ActionInfo> actionsInfo = new ArrayList<>();
 803         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 804                 routerName);
 805         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 806         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 807         actionsInfo.add(new ActionGroup(groupId));
 808         List<InstructionInfo> instructions = new ArrayList<>();
 809         instructions.add(new InstructionApplyActions(actionsInfo));
 810         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 811         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 812                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 813                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 814
 815         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 816         return flowEntity;
 817     }
 818
 819     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 820
 821         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 822             routerName);
 823         List<MatchInfo> matches = new ArrayList<>();
 824         matches.add(MatchEthernetType.IPV4);
 825         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 826
 827         List<InstructionInfo> instructions = new ArrayList<>();
 828         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 829
 830         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 831         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 832             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 833             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 834
 835         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 836         return flowEntity;
 837     }
 838
 839     // TODO : Replace this with ITM Rpc once its available with full functionality
 840     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 841         TypedWriteTransaction<Configuration> confTx) {
 842
 843         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 844             dpnId, routerName);
 845         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 846         if (flowEntity == null) {
 847             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 848                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 849                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 850                     dpnId, routerName);
 851             return;
 852         }
 853         mdsalManager.addFlow(confTx, flowEntity);
 854
 855     }
 856
 857     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 858         List<MatchInfo> matches = new ArrayList<>();
 859         matches.add(MatchEthernetType.IPV4);
 860         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 861                 routerName);
 862         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 863         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 864         List<InstructionInfo> instructions = new ArrayList<>();
 865         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 866                 MetaDataUtil.METADATA_MASK_VRFID));
 867         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 868         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 869                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 870                 instructions);
 871         return flowEntity;
 872     }
 873
 874     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 875         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 876                 .FLOWID_SEPARATOR + routerID;
 877     }
 878
 879     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 880         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 881                 .FLOWID_SEPARATOR + routerID;
 882     }
 883
 884     private String getGroupIdKey(String routerName) {
 885         return "snatmiss." + routerName;
 886     }
 887
 888     protected long createGroupId(String groupIdKey) {
 889         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 890             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 891             .build();
 892         try {
 893             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 894             RpcResult<AllocateIdOutput> rpcResult = result.get();
 895             return rpcResult.getResult().getIdValue();
 896         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 897             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 898         }
 899         return 0;
 900     }
 901
 902     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 903         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 904         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 905         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 906         List<BucketInfo> listBucketInfo = new ArrayList<>();
 907
 908         if (ifNamePrimary != null) {
 909             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 910             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 911                     interfaceManager, ifNamePrimary, routerId, true);
 912             if (listActionInfoPrimary.isEmpty()) {
 913                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 914                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 915                         ifNamePrimary);
 916             }
 917         } else {
 918             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 919                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 920         }
 921         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 922
 923         listBucketInfo.add(0, bucketPrimary);
 924         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 925     }
 926
 927     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 928                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 929         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 930         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 931         List<BucketInfo> listBucketInfo = new ArrayList<>();
 932
 933         if (ifNamePrimary != null) {
 934             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 935                     ifNamePrimary);
 936             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 937                     interfaceManager, ifNamePrimary, routerId, true);
 938             if (listActionInfoPrimary.isEmpty()) {
 939                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 940                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 941                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 942             }
 943         } else {
 944             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 945                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 946         }
 947         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 948
 949         listBucketInfo.add(0, bucketPrimary);
 950         return listBucketInfo;
 951     }
 952
 953     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 954         TypedWriteTransaction<Configuration> confTx) {
 955
 956        /*
 957         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 958         */
 959
 960         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 961
 962 /*
 963         List<BucketInfo> listBucketInfo = new ArrayList<>();
 964         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 965         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 966         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 967         listBucketInfo.add(0, bucketPrimary);
 968 */
 969
 970         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 971         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 972         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 973         installNaptPfibEntry(dpnId, routerId, confTx);
 974         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 975         if (networkId != null) {
 976             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 977             if (vpnUuid != null) {
 978                 Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 979                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 980                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 981                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 982                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 983                         installNaptPfibEntry(dpnId, vpnId, null);
 984                     }
 985                     return Collections.emptyList();
 986                 });
 987             } else {
 988                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 989             }
 990         } else {
 991             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 992         }
 993     }
 994
 995     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 996         List<BucketInfo> listBucketInfo = new ArrayList<>();
 997         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 998         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 999         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
1000         listBucketInfo.add(0, bucketPrimary);
1001         return listBucketInfo;
1002     }
1003
1004     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, TypedWriteTransaction<Configuration> confTx) {
1005         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1006         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1007         if (confTx != null) {
1008             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1009         } else {
1010             mdsalManager.installFlow(naptPfibFlowEntity);
1011         }
1012     }
1013
1014     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1015
1016         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1017         List<MatchInfo> matches = new ArrayList<>();
1018         matches.add(MatchEthernetType.IPV4);
1019         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1020
1021         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1022         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1023         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1024         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1025         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1026
1027         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1028         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1029             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1030             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1031         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1032         return flowEntity;
1033     }
1034
1035     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1036         long routerId, String routerName, String externalIp) {
1037         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1038             dpnId, routerId, externalIp);
1039         Uuid networkId = router.getNetworkId();
1040         if (networkId == null) {
1041             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1042             return;
1043         }
1044         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1045         if (vpnName == null) {
1046             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1047                     + "configuration {} in router {}", networkId, externalIp, routerId);
1048             return;
1049         }
1050         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1051             externalIp, networkId, router, confTx);
1052         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1053             dpnId, routerId, externalIp);
1054     }
1055
1056     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1057                                                 final long routerId, final String routerName, final String externalIp,
1058                                                 final Uuid extNetworkId, final Routers router,
1059                                                 final TypedWriteTransaction<Configuration> confTx) {
1060         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1061                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1062         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1063         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1064         if (rd == null || rd.isEmpty()) {
1065             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1066             return;
1067         }
1068         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1069         if (extNwProvType == null) {
1070             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1071             return;
1072         }
1073         if (extNwProvType == ProviderTypes.VXLAN) {
1074             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1075                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1076             return;
1077         }
1078         //Generate VPN label for the external IP
1079         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1080             .setIpPrefix(externalIp).build();
1081         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1082
1083         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1084         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1085             if (result.isSuccessful()) {
1086                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1087                 GenerateVpnLabelOutput output = result.getResult();
1088                 final long label = output.getLabel();
1089
1090                 int externalIpInDsFlag = 0;
1091                 //Get IPMaps from the DB for the router ID
1092                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1093                 if (dbIpMaps != null) {
1094                     for (IpMap dbIpMap : dbIpMaps) {
1095                         String dbExternalIp = dbIpMap.getExternalIp();
1096                         //Select the IPMap, whose external IP is the IP for which FIB is installed
1097                         if (dbExternalIp.contains(externalIp)) {
1098                             String dbInternalIp = dbIpMap.getInternalIp();
1099                             IpMapKey dbIpMapKey = dbIpMap.key();
1100                             LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1101                                     + "and externalIp {}", label, dbInternalIp, externalIp);
1102                             IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1103                                 .setExternalIp(dbExternalIp).setLabel(label).build();
1104                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1105                                 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1106                             externalIpInDsFlag++;
1107                         }
1108                     }
1109                     if (externalIpInDsFlag <= 0) {
1110                         LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1111                                 + "Failed to update label {} for routerId {} in DS",
1112                                 externalIp, label, routerId);
1113                         String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1114                             + " found in DS for router %s", label, externalIp, routerId);
1115                         return Futures.immediateFailedFuture(new Exception(errMsg));
1116                     }
1117                 } else {
1118                     LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1119                             + " routerId {} in DS", label, externalIp, routerId);
1120                 }
1121                 //Inform BGP
1122                 long l3vni = 0;
1123                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1124                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1125                 }
1126                 Routers extRouter = router != null ? router :
1127                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1128                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1129                         extRouter);
1130                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1131                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1132                     RouteOrigin.STATIC, dpnId);
1133
1134                 //Install custom FIB routes
1135                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1136                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1137                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1138                         extNwProvType);
1139                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1140
1141                 //Install custom FIB routes - FIB table.
1142                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1143                         routerName, externalIp);
1144                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1145                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1146                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1147                 }
1148                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1149                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1150                         externalSubnetId);
1151                 String externalVpn = vpnName;
1152                 if (externalSubnet.isPresent()) {
1153                     externalVpn =  externalSubnetId.getValue();
1154                 }
1155                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1156                     .setVpnName(externalVpn)
1157                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1158                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1159                     .setInstruction(fibTableCustomInstructions).build();
1160                 return fibService.createFibEntry(input);
1161             } else {
1162                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1163                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1164                     externalIp, vpnName, result.getErrors());
1165                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1166             }
1167         }, MoreExecutors.directExecutor());
1168
1169         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1170
1171             @Override
1172             public void onFailure(@Nonnull Throwable error) {
1173                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1174             }
1175
1176             @Override
1177             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1178                 if (result.isSuccessful()) {
1179                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1180                             externalIp);
1181                 } else {
1182                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1183                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1184                 }
1185             }
1186         }, MoreExecutors.directExecutor());
1187     }
1188
1189     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1190             String externalIp) {
1191         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1192         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1193         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1194                 externalIp, router);
1195         int instructionIndex = 0;
1196         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1197             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1198             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1199                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1200             instructionIndex++;
1201         }
1202
1203         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1204         return fibTableCustomInstructions;
1205     }
1206
1207     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1208         TypedWriteTransaction<Configuration> confTx) {
1209         List<MatchInfo> matches = new ArrayList<>();
1210         matches.add(MatchEthernetType.MPLS_UNICAST);
1211         matches.add(new MatchMplsLabel(serviceId));
1212
1213         List<Instruction> instructions = new ArrayList<>();
1214         List<ActionInfo> actionsInfos = new ArrayList<>();
1215         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1216         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1217         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1218         instructions.add(writeInstruction);
1219         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1220
1221         // Install the flow entry in L3_LFIB_TABLE
1222         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1223
1224         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1225             10, flowRef, 0, 0,
1226             COOKIE_VM_LFIB_TABLE, matches, instructions);
1227
1228         mdsalManager.addFlow(confTx, dpId, flowEntity);
1229
1230         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1231     }
1232
1233     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1234         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1235         ProviderTypes extNwProvType) {
1236
1237         List<MatchInfo> mkMatches = new ArrayList<>();
1238
1239         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1240             dpnId, serviceId, customInstructions);
1241
1242         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1243             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1244         } else {
1245             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1246         }
1247
1248         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1249             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1250             String.format("%s:%d", "TST Flow Entry ", serviceId),
1251             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1252
1253         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1254     }
1255
1256     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1257         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1258             new RouterIdsKey(routerId)).build();
1259     }
1260
1261     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1262         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1263                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1264     }
1265
1266     @Override
1267     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1268         String routerName = original.getRouterName();
1269         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1270         if (routerId == NatConstants.INVALID_ID) {
1271             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1272             return;
1273         }
1274         // Check if its update on SNAT flag
1275         boolean originalSNATEnabled = original.isEnableSnat();
1276         boolean updatedSNATEnabled = update.isEnableSnat();
1277         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1278             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1279         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1280          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1281          */
1282         long bgpVpnId = NatConstants.INVALID_ID;
1283         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1284         if (bgpVpnUuid != null) {
1285             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1286         }
1287         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1288         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1289             // Router has no interface attached
1290             return;
1291         }
1292         final long finalBgpVpnId = bgpVpnId;
1293         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1294             List<ListenableFuture<Void>> futures = new ArrayList<>();
1295             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1296                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1297                     Uuid networkId = original.getNetworkId();
1298                     if (originalSNATEnabled != updatedSNATEnabled) {
1299                         if (originalSNATEnabled) {
1300                             //SNAT disabled for the router
1301                             Uuid networkUuid = original.getNetworkId();
1302                             LOG.info("update : SNAT disabled for Router {}", routerName);
1303                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1304                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1305                                     removeFlowInvTx);
1306                         } else {
1307                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1308                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1309                                     true, writeFlowInvTx);
1310                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1311                         }
1312                     }
1313                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1314                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1315                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1316                     }
1317
1318                     //Check if the Update is on External IPs
1319                     LOG.debug("update : Checking if this is update on External IPs");
1320                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1321                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1322
1323                     //Check if the External IPs are added during the update.
1324                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1325                     addedExternalIps.removeAll(originalExternalIps);
1326                     if (addedExternalIps.size() != 0) {
1327                         LOG.debug("update : Start processing of the External IPs addition during the update "
1328                                 + "operation");
1329                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1330                                 update.getExtGwMacAddress(), dpnId,
1331                                 update.getNetworkId());
1332
1333                         for (String addedExternalIp : addedExternalIps) {
1334                 /*
1335                     1) Do nothing in the IntExtIp model.
1336                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1337                  */
1338                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1339                             String externalIp = externalIpParts[0];
1340                             String externalIpPrefix = externalIpParts[1];
1341                             String externalpStr = externalIp + "/" + externalIpPrefix;
1342                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1343                                             + "router ID {} in the ExternalIpsCounter model.",
1344                                     externalpStr, routerId);
1345                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1346                         }
1347                         LOG.debug(
1348                                 "update : End processing of the External IPs addition during the update operation");
1349                     }
1350
1351                     //Check if the External IPs are removed during the update.
1352                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1353                     removedExternalIps.removeAll(updatedExternalIps);
1354                     if (removedExternalIps.size() > 0) {
1355                         LOG.debug("update : Start processing of the External IPs removal during the update "
1356                                 + "operation");
1357                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1358                                 removedExternalIps, original.getExtGwMacAddress(),
1359                                 dpnId, networkId);
1360
1361                         for (String removedExternalIp : removedExternalIps) {
1362                 /*
1363                     1) Remove the mappings in the IntExt IP model which has external IP.
1364                     2) Remove the external IP in the ExternalCounter model.
1365                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1366                        least loaded external IP.
1367                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1368                     4) Increase the count of the allocated external IP by one.
1369                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1370                      i.e. the route for the external IP is absent.
1371                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1372                      the removed external IPs and also from the model.
1373                     7) Advertise to the BGP for removing the route for the removed external IPs.
1374                  */
1375
1376                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1377                             String externalIp = externalIpParts[0];
1378                             String externalIpPrefix = externalIpParts[1];
1379                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1380
1381                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1382                                     + "entries for removed external IP {}", externalIpAddrStr);
1383                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1384                             String vpnName = "";
1385                             if (vpnUuId != null) {
1386                                 vpnName = vpnUuId.getValue();
1387                             }
1388                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1389                                     update.getExtGwMacAddress(), removeFlowInvTx);
1390
1391                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1392                             //Get the internal IPs which are associated to the removed external IPs
1393                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1394                             List<String> removedInternalIps = new ArrayList<>();
1395                             for (IpMap ipMap : ipMaps) {
1396                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1397                                     removedInternalIps.add(ipMap.getInternalIp());
1398                                 }
1399                             }
1400
1401                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1402                             for (String removedInternalIp : removedInternalIps) {
1403                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1404                                                 + "router ID {} from the IntExtIP model",
1405                                         removedInternalIp, routerId);
1406                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1407                             }
1408
1409                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1410                                             + "router ID {} from the ExternalIpsCounter model.",
1411                                     externalIpAddrStr, routerId);
1412                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1413
1414                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1415                                     + "whose external IPs were removed.");
1416                             for (String removedInternalIp : removedInternalIps) {
1417                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1418                                         removedInternalIp, writeFlowInvTx);
1419                             }
1420
1421                             LOG.debug("update : Remove the NAPT translation entries from "
1422                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1423                             //Get the internalIP and internal Port which were associated to the removed external IP.
1424                             List<Integer> externalPorts = new ArrayList<>();
1425                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1426                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1427                                     .builder(IntextIpPortMap.class)
1428                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1429                             Optional<IpPortMapping> ipPortMapping =
1430                                     MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1431                             if (ipPortMapping.isPresent()) {
1432                                 List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1433                                         .getIntextIpProtocolType();
1434                                 for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1435                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1436                                     List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1437                                     for (IpPortMap ipPortMap : ipPortMaps) {
1438                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1439                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1440                                             externalPorts.add(ipPortExternal.getPortNum());
1441                                             List<String> removedInternalIpPorts =
1442                                                     protoTypesIntIpPortsMap.get(protoType);
1443                                             if (removedInternalIpPorts != null) {
1444                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1445                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1446                                             } else {
1447                                                 removedInternalIpPorts = new ArrayList<>();
1448                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1449                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1450                                             }
1451                                         }
1452                                     }
1453                                 }
1454                             }
1455
1456                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1457                             // the removed external IP.
1458                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1459                                     protoTypesIntIpPortsMap.entrySet();
1460                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1461                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1462                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1463                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1464                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1465                                     // Remove the IP port map from the intext-ip-port-map model,
1466                                     // which were containing the removed external IP
1467                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1468                                             protocolType);
1469                                     //Remove the IP port incomint packer map.
1470                                     naptPacketInHandler.removeIncomingPacketMap(
1471                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1472                                     String[] removedInternalIpPortParts = removedInternalIpPort
1473                                             .split(NatConstants.COLON_SEPARATOR);
1474                                     if (removedInternalIpPortParts.length == 2) {
1475                                         String removedInternalIp = removedInternalIpPortParts[0];
1476                                         String removedInternalPort = removedInternalIpPortParts[1];
1477                                         List<String> removedInternalPortsList =
1478                                                 internalIpPortMap.get(removedInternalPort);
1479                                         if (removedInternalPortsList != null) {
1480                                             removedInternalPortsList.add(removedInternalPort);
1481                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1482                                         } else {
1483                                             removedInternalPortsList = new ArrayList<>();
1484                                             removedInternalPortsList.add(removedInternalPort);
1485                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1486                                         }
1487                                     }
1488                                 }
1489                             }
1490
1491                             // Delete the entry from SnatIntIpPortMap DS
1492                             Set<String> internalIps = internalIpPortMap.keySet();
1493                             for (String internalIp : internalIps) {
1494                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1495                                         + "model SnatIntIpPortMap", internalIp);
1496                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1497                             }
1498
1499                             naptManager.removeNaptPortPool(externalIp);
1500
1501                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1502                                     + "the removed external IP {}", externalIp);
1503                             for (Integer externalPort : externalPorts) {
1504                                 //Remove the NAPT translation entries from Inbound NAPT table
1505                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1506                                         routerId, externalIp, externalPort);
1507                             }
1508
1509                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1510                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1511                                 String internalIp = internalIpPort.getKey();
1512                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1513                                         + "for the removed internal IP {}", internalIp);
1514                                 List<String> internalPorts = internalIpPort.getValue();
1515                                 for (String internalPort : internalPorts) {
1516                                     //Remove the NAPT translation entries from Outbound NAPT table
1517                                     naptPacketInHandler.removeIncomingPacketMap(
1518                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1519                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1520                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1521                                             routerId, internalIp, Integer.parseInt(internalPort));
1522                                 }
1523                             }
1524                         }
1525                         LOG.debug(
1526                                 "update : End processing of the External IPs removal during the update operation");
1527                     }
1528
1529                     //Check if its Update on subnets
1530                     LOG.debug("update : Checking if this is update on subnets");
1531                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1532                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1533                     Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1534                     addedSubnetIds.removeAll(originalSubnetIds);
1535
1536                     //Check if the Subnet IDs are added during the update.
1537                     if (addedSubnetIds.size() != 0) {
1538                         LOG.debug(
1539                                 "update : Start processing of the Subnet IDs addition during the update operation");
1540                         for (Uuid addedSubnetId : addedSubnetIds) {
1541                 /*
1542                     1) Select the least loaded external IP for the subnet and store the mapping of the
1543                     subnet IP and the external IP in the IntExtIp model.
1544                     2) Increase the count of the selected external IP by one.
1545                     3) Advertise to the BGP if external IP is allocated for the first time for the
1546                     router i.e. the route for the external IP is absent.
1547                  */
1548                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1549                             if (subnetIp != null) {
1550                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1551                                         writeFlowInvTx);
1552                             }
1553                         }
1554                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1555                     }
1556
1557                     //Check if the Subnet IDs are removed during the update.
1558                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1559                     removedSubnetIds.removeAll(updatedSubnetIds);
1560                     if (removedSubnetIds.size() != 0) {
1561                         LOG.debug(
1562                                 "update : Start processing of the Subnet IDs removal during the update operation");
1563                         for (Uuid removedSubnetId : removedSubnetIds) {
1564                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1565                             if (subnetAddr != null) {
1566                     /*
1567                         1) Remove the subnet IP and the external IP in the IntExtIp map
1568                         2) Decrease the count of the coresponding external IP by one.
1569                         3) Advertise to the BGP for removing the routes of the corresponding external
1570                         IP if its not allocated to any other internal IP.
1571                     */
1572
1573                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1574                                         subnetAddr[0] + "/" + subnetAddr[1]);
1575                                 if (externalIp == null) {
1576                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1577                                             routerId, subnetAddr[0]);
1578                                     return;
1579                                 }
1580
1581                                 naptManager.updateCounter(routerId, externalIp, false);
1582                                 // Traverse entire model of external-ip counter whether external ip is not
1583                                 // used by any other internal ip in any router
1584                                 if (!isExternalIpAllocated(externalIp)) {
1585                                     LOG.debug("update : external ip is not allocated to any other "
1586                                             + "internal IP so proceeding to remove routes");
1587                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1588                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1589                                             removeFlowInvTx);
1590                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1591                                                     + "router {} with networkId {} and externalIp {}",
1592                                             dpnId, routerId, networkId, externalIp);
1593                                 }
1594
1595                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1596                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1597                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1598                             }
1599                         }
1600                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1601                     }
1602                 }));
1603             }));
1604             return futures;
1605         }, NatConstants.NAT_DJC_MAX_RETRIES);
1606     }
1607
1608     private boolean isExternalIpAllocated(String externalIp) {
1609         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1610         Optional<ExternalIpsCounter> externalCountersData =
1611             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1612         if (externalCountersData.isPresent()) {
1613             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1614             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1615             for (ExternalCounters ext : externalCounters) {
1616                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1617                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1618                         if (externalIpCount.getCounter() != 0) {
1619                             return true;
1620                         }
1621                         break;
1622                     }
1623                 }
1624             }
1625         }
1626         return false;
1627     }
1628
1629     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1630             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1631         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1632         try {
1633             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1634             if (address instanceof Inet6Address) {
1635                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1636                 return;
1637             }
1638         } catch (UnknownHostException e) {
1639             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1640             return;
1641         }
1642         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1643         if (leastLoadedExtIpAddr != null) {
1644             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1645             String leastLoadedExtIp = externalIpParts[0];
1646             String leastLoadedExtIpPrefix = externalIpParts[1];
1647             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1648             subnetIp = subnetIpParts[0];
1649             String subnetIpPrefix = subnetIpParts[1];
1650             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1651             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1652                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1653                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1654             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1655
1656
1657             // Check if external IP is already assigned a route. (i.e. External IP is previously
1658             // allocated to any of the subnets)
1659             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1660             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1661             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1662             if (label != null) {
1663                 //update
1664                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1665                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1666                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1667                     label, internalIp, leastLoadedExtIpAddrStr);
1668                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1669                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1670                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1671                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1672                 return;
1673             }
1674
1675             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1676             // for the first time and hence not having a route.
1677             //Get the VPN Name using the network ID
1678             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1679             if (vpnName != null) {
1680                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1681                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1682                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1683                         + "added after gateway-set");
1684                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1685                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1686                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1687                         return;
1688                     }
1689                 }
1690                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1691                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1692                     writeFlowInvTx);
1693             }
1694         }
1695     }
1696
1697     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1698         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1699         for (IpMap ipMap : ipMaps) {
1700             if (ipMap.getExternalIp().equals(externalIp)) {
1701                 if (ipMap.getLabel() != null) {
1702                     return ipMap.getLabel();
1703                 }
1704             }
1705         }
1706         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1707         return null;
1708     }
1709
1710     @Override
1711     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1712         LOG.trace("remove : Router delete method");
1713         /*
1714         ROUTER DELETE SCENARIO
1715         1) Get the router ID from the event.
1716         2) Build the cookie information from the router ID.
1717         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1718         4) Build the flow with the cookie value.
1719         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1720         6) Remove the flows from the other switches which points to the primary and secondary
1721          switches for the flows related the router ID.
1722         7) Get the list of external IP address maintained for the router ID.
1723         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1724         9) Withdraw the corresponding routes from the BGP.
1725          */
1726
1727         if (identifier == null || router == null) {
1728             LOG.error("remove : returning without processing since routers is null");
1729             return;
1730         }
1731
1732         String routerName = router.getRouterName();
1733         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1734             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1735                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1736                         routerName);
1737                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1738                 if (routerId == NatConstants.INVALID_ID) {
1739                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1740                             routerName);
1741                     return;
1742                 }
1743                 long bgpVpnId = NatConstants.INVALID_ID;
1744                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1745                 if (bgpVpnUuid != null) {
1746                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1747                 }
1748                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1749                         tx);
1750                 Uuid networkUuid = router.getNetworkId();
1751
1752                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1753                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1754                     // No NAPT switch for external router, probably because the router is not attached to
1755                     // any
1756                     // internal networks
1757                     LOG.debug(
1758                             "No NAPT switch for router {}, check if router is attached to any internal "
1759                                     + "network",
1760                             routerName);
1761                     return;
1762                 } else {
1763                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1764                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1765                             routerId, tx);
1766                 }
1767                 NatOverVxlanUtil.releaseVNI(routerName, idManager);
1768             })), NatConstants.NAT_DJC_MAX_RETRIES);
1769     }
1770
1771     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1772                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1773                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1774         LOG.info("handleDisableSnat : Entry");
1775         String routerName = router.getRouterName();
1776         try {
1777             if (routerFlag) {
1778                 removeNaptSwitch(routerName);
1779             } else {
1780                 updateNaptSwitch(routerName, BigInteger.ZERO);
1781             }
1782
1783             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1784             naptManager.removeExternalCounter(routerId);
1785
1786             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1787             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1788                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1789                     + "router ID {} from RouterNaptSwitch model", routerId);
1790                 return;
1791             }
1792             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1793                     networkUuid);
1794             if (extNwProvType == null) {
1795                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1796                 return;
1797             }
1798             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1799             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1800                     externalSubnetList, removeFlowInvTx, extNwProvType);
1801             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1802             String externalSubnetVpn = null;
1803             for (Uuid externalSubnetId : externalSubnetList) {
1804                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1805                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1806                 if (externalSubnet.isPresent()) {
1807                     externalSubnetVpn =  externalSubnetId.getValue();
1808                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1809                             router.getExtGwMacAddress(), removeFlowInvTx);
1810                 }
1811             }
1812             if (externalSubnetVpn == null) {
1813                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1814                         router.getExtGwMacAddress(), removeFlowInvTx);
1815             }
1816             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1817             // for the router ID.
1818             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1819                     + "router ID {} in the DS", routerId);
1820             naptManager.removeMapping(routerId);
1821         } catch (InterruptedException | ExecutionException e) {
1822             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1823         }
1824         LOG.info("handleDisableSnat : Exit");
1825     }
1826
1827     // TODO Clean up the exception handling
1828     @SuppressWarnings("checkstyle:IllegalCatch")
1829     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1830                                              @Nonnull Collection<String> externalIps,
1831                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1832         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1833                 + "with internet vpn {}", routerName, vpnId);
1834         try {
1835             BigInteger naptSwitchDpnId = null;
1836             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1837                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1838             Optional<RouterToNaptSwitch> rtrToNapt =
1839                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1840             if (rtrToNapt.isPresent()) {
1841                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1842             }
1843             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1844
1845             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1846                     writeFlowInvTx);
1847             try {
1848                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1849                 if (extGwMacAddress != null) {
1850                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1851                             + "External Router ID {}", extGwMacAddress, routerId);
1852                 } else {
1853                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1854                             + "External Router ID {}", routerId);
1855                     return;
1856                 }
1857                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1858                         writeFlowInvTx);
1859             } catch (Exception ex) {
1860                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1861                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1862             }
1863             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1864         } catch (InterruptedException | ExecutionException e) {
1865             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1866                     + "with internet vpn {}", routerName, vpnId, e);
1867         }
1868         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1869                 routerName, vpnId);
1870     }
1871
1872     // TODO Clean up the exception handling
1873     @SuppressWarnings("checkstyle:IllegalCatch")
1874     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1875         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1876             .setPrimarySwitchId(naptSwitchId).build();
1877         try {
1878             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1879                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1880         } catch (Exception ex) {
1881             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1882                 naptSwitchId, routerName);
1883         }
1884         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1885             naptSwitchId, routerName);
1886     }
1887
1888     protected void removeNaptSwitch(String routerName) {
1889         // Remove router and switch from model
1890         InstanceIdentifier<RouterToNaptSwitch> id =
1891             InstanceIdentifier.builder(NaptSwitches.class)
1892                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1893         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1894         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1895         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1896         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1897     }
1898
1899     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1900                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1901                                                 @Nonnull Collection<String> externalIps,
1902                                                 Collection<Uuid> externalSubnetList,
1903                                                 TypedReadWriteTransaction<Configuration> confTx,
1904                                                 ProviderTypes extNwProvType)
1905             throws InterruptedException, ExecutionException {
1906
1907         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1908         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1909
1910         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1911         // traffic which comes from the  VMs of the NAPT switches)
1912         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1913         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1914
1915         LOG.info(
1916             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1917                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1918         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1919
1920         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1921         // traffic which comes from the VMs of the non NAPT switches)
1922         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1923             routerName);
1924         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1925         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1926         LOG.info(
1927             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1928                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1929         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1930
1931         //Remove the flow table 25->44 from NAPT Switch
1932         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1933             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1934         }
1935
1936         //Remove the Outbound flow entry which forwards the packet to FIB Table
1937         LOG.info(
1938             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1939                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1940
1941         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1942             NwConstants.IP_PROT_TCP);
1943         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1944             outboundTcpNatFlowRef);
1945         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1946
1947         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1948             NwConstants.IP_PROT_UDP);
1949         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1950             outboundUdpNatFlowRef);
1951         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1952
1953         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1954             NwConstants.IP_PROT_ICMP);
1955         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1956             icmpDropFlowRef);
1957         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1958         boolean lastRouterOnExternalNetwork =
1959             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1960         if (lastRouterOnExternalNetwork) {
1961             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1962         }
1963         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1964         // External Subnet Vpn Id.
1965         for (Uuid externalSubnetId : externalSubnetList) {
1966             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1967             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1968                 dataBroker, externalSubnetId, routerName, dpnId)) {
1969                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1970                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1971                     natPfibSubnetFlowRef);
1972                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1973                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1974                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1975                     subnetVpnId, dpnId);
1976             }
1977         }
1978
1979         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1980         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1981         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1982
1983         LOG.info(
1984             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1985             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1986         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1987
1988         if (lastRouterOnExternalNetwork) {
1989             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1990             // - This does not work since ext-routers is deleted already - no network info
1991             //Get the VPN ID from the ExternalNetworks model
1992             long vpnId = -1;
1993             if (vpnName == null || vpnName.isEmpty()) {
1994                 // ie called from router delete cases
1995                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1996                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1997                 if (vpnUuid != null) {
1998                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1999                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2000                               + "or disableSNAT scenario", vpnId, networkId);
2001                 }
2002             } else {
2003                 // ie called from disassociate vpn case
2004                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2005                     vpnName);
2006                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2007                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2008                     vpnId);
2009             }
2010
2011             if (vpnId != NatConstants.INVALID_ID) {
2012                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2013                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2014                 FlowEntity natPfibVpnFlowEntity =
2015                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2016                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2017                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2018                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2019             }
2020         }
2021
2022         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2023         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2024         if (ipPortMapping == null) {
2025             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2026             return;
2027         }
2028
2029         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2030         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2031             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2032             for (IpPortMap ipPortMap : ipPortMaps) {
2033                 String ipPortInternal = ipPortMap.getIpPortInternal();
2034                 String[] ipPortParts = ipPortInternal.split(":");
2035                 if (ipPortParts.length != 2) {
2036                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2037                     return;
2038                 }
2039                 String internalIp = ipPortParts[0];
2040                 String internalPort = ipPortParts[1];
2041
2042                 //Build the flow for the outbound NAPT table
2043                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2044                     + NatConstants.COLON_SEPARATOR + internalPort);
2045                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2046                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2047                 FlowEntity outboundNaptFlowEntity =
2048                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2049
2050                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2051                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2052                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2053
2054                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2055                 String externalIp = ipPortExternal.getIpAddress();
2056                 int externalPort = ipPortExternal.getPortNum();
2057
2058                 //Build the flow for the inbound NAPT table
2059                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2060                     String.valueOf(routerId), externalIp, externalPort);
2061                 FlowEntity inboundNaptFlowEntity =
2062                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2063
2064                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2065                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2066                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2067             }
2068         }
2069     }
2070
2071     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2072                                                     @Nonnull Collection<String> externalIps,
2073                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2074             throws ExecutionException, InterruptedException {
2075         long extVpnId = NatConstants.INVALID_ID;
2076         if (networkId != null) {
2077             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2078             if (vpnUuid != null) {
2079                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2080             } else {
2081                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2082             }
2083         } else {
2084             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2085             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2086         }
2087         if (extVpnId == NatConstants.INVALID_ID) {
2088             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2089             extVpnId = routerId;
2090         }
2091         for (String ip : externalIps) {
2092             String extIp = removeMaskFromIp(ip);
2093             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2094             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2095                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2096                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2097             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2098             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2099         }
2100     }
2101
2102     private String removeMaskFromIp(String ip) {
2103         if (ip != null && !ip.trim().isEmpty()) {
2104             return ip.split("/")[0];
2105         }
2106         return ip;
2107     }
2108
2109     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2110                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2111                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2112             throws ExecutionException, InterruptedException {
2113         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2114         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2115
2116         //Remove the NAPT PFIB TABLE entry
2117         long vpnId = -1;
2118         if (vpnName != null) {
2119             // ie called from disassociate vpn case
2120             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2121                     + "with vpnName {}", vpnName);
2122             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2123             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2124                     vpnId);
2125         }
2126
2127         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2128                 networkId, routerName, dpnId)) {
2129             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2130             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2131             FlowEntity natPfibVpnFlowEntity =
2132                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2133             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2134                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2135             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2136
2137             // Remove IP-PORT active NAPT entries and release port from IdManager
2138             // For the router ID get the internal IP , internal port and the corresponding
2139             // external IP and external Port.
2140             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2141             if (ipPortMapping == null) {
2142                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2143                 return;
2144             }
2145             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2146             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2147                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2148                 for (IpPortMap ipPortMap : ipPortMaps) {
2149                     String ipPortInternal = ipPortMap.getIpPortInternal();
2150                     String[] ipPortParts = ipPortInternal.split(":");
2151                     if (ipPortParts.length != 2) {
2152                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2153                                 + "and port");
2154                         return;
2155                     }
2156                     String internalIp = ipPortParts[0];
2157                     String internalPort = ipPortParts[1];
2158
2159                     //Build the flow for the outbound NAPT table
2160                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2161                             + NatConstants.COLON_SEPARATOR + internalPort);
2162                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2163                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2164                     FlowEntity outboundNaptFlowEntity =
2165                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2166
2167                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2168                             + "active switch with the DPN ID {} and router ID {}",
2169                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2170                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2171
2172                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2173                     String externalIp = ipPortExternal.getIpAddress();
2174                     int externalPort = ipPortExternal.getPortNum();
2175
2176                     //Build the flow for the inbound NAPT table
2177                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2178                         String.valueOf(routerId), externalIp, externalPort);
2179                     FlowEntity inboundNaptFlowEntity =
2180                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2181
2182                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2183                             + "active active switch with the DPN ID {} and router ID {}",
2184                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2185                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2186
2187                     // Finally release port from idmanager
2188                     String internalIpPort = internalIp + ":" + internalPort;
2189                     naptManager.removePortFromPool(internalIpPort, externalIp);
2190
2191                     //Remove sessions from models
2192                     naptManager.removeIpPortMappingForRouterID(routerId);
2193                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2194                 }
2195             }
2196         } else {
2197             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2198         }
2199     }
2200
2201     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2202             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2203             throws ExecutionException, InterruptedException {
2204         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2205
2206         // Remove the flows from the other switches which points to the primary and secondary switches
2207         // for the flows related the router ID.
2208         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2209         if (allSwitchList.isEmpty()) {
2210             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2211             return;
2212         }
2213         for (BigInteger dpnId : allSwitchList) {
2214             if (!naptSwitchDpnId.equals(dpnId)) {
2215                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2216
2217                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2218                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2219                 FlowEntity preSnatFlowEntity =
2220                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2221
2222                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2223                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2224                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2225
2226                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2227                 long groupId = createGroupId(getGroupIdKey(routerName));
2228
2229                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2230                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2231                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2232             }
2233         }
2234     }
2235
2236     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2237                                          @Nonnull Collection<String> externalIps, String vpnName,
2238                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2239             throws ExecutionException, InterruptedException {
2240         //Withdraw the corresponding routes from the BGP.
2241         //Get the network ID using the router ID.
2242         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2243                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2244         if (networkUuid == null) {
2245             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2246             return;
2247         }
2248
2249         if (externalIps.isEmpty()) {
2250             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2251             return;
2252         }
2253
2254         if (vpnName == null) {
2255             //Get the VPN Name using the network ID
2256             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2257             if (vpnName == null) {
2258                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2259                     networkUuid, routerId);
2260                 return;
2261             }
2262         }
2263         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2264
2265         //Remove custom FIB routes
2266         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2267         for (String extIp : externalIps) {
2268             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2269         }
2270     }
2271
2272     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2273                                             final Uuid networkUuid, String extGwMacAddress,
2274                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2275             throws ExecutionException, InterruptedException {
2276         clearBgpRoutes(extIp, vpnName);
2277         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2278                 removeFlowInvTx);
2279     }
2280
2281     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2282                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2283                                              String gwMacAddress, boolean switchOver,
2284                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2285             throws ExecutionException, InterruptedException {
2286         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2287         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2288         if (routerName == null) {
2289             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2290             return;
2291         }
2292         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2293         if (extNwProvType == null) {
2294             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2295             return;
2296         }
2297         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2298          * external network provided type is VxLAN
2299          */
2300         if (extNwProvType == ProviderTypes.VXLAN) {
2301             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2302             );
2303             return;
2304         }
2305         if (tempLabel < 0) {
2306             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2307                     extIp, routerId);
2308             return;
2309         }
2310
2311         final long label = tempLabel;
2312         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2313         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2314                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2315                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2316         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2317
2318         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2319         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2320         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2321             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2322             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2323         }
2324         if (!switchOver) {
2325             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2326                 Futures.transformAsync(future, result -> {
2327                     //Release label
2328                     if (result.isSuccessful()) {
2329                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2330                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2331                             .setVpnName(vpnName).setIpPrefix(externalIp).build();
2332                         return vpnService.removeVpnLabel(labelInput);
2333                     } else {
2334                         String errMsg =
2335                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2336                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2337                         LOG.error(errMsg);
2338                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2339                     }
2340                 }, MoreExecutors.directExecutor());
2341
2342             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2343
2344                 @Override
2345                 public void onFailure(@Nonnull Throwable error) {
2346                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2347                         + "got external ip {}", label, extIp, error);
2348                 }
2349
2350                 @Override
2351                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2352                     if (result.isSuccessful()) {
2353                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2354                             + "from VPN {}", externalIp, vpnName);
2355                     } else {
2356                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2357                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2358                     }
2359                 }
2360             }, MoreExecutors.directExecutor());
2361         } else {
2362             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2363                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2364         }
2365     }
2366
2367     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2368                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2369                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2370             throws ExecutionException, InterruptedException {
2371         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2372         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2373         if (routerName == null) {
2374             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2375             return;
2376         }
2377         //Get the external network provider type from networkId
2378         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2379         if (extNwProvType == null) {
2380             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2381                     networkUuid);
2382             return;
2383         }
2384         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2385          *  external network provided type is VxLAN
2386          */
2387         if (extNwProvType == ProviderTypes.VXLAN) {
2388             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2389             return;
2390         }
2391         //Get IPMaps from the DB for the router ID
2392         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2393         if (dbIpMaps.isEmpty()) {
2394             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2395             return;
2396         }
2397
2398         long tempLabel = NatConstants.INVALID_ID;
2399         for (IpMap dbIpMap : dbIpMaps) {
2400             String dbExternalIp = dbIpMap.getExternalIp();
2401             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2402             //Select the IPMap, whose external IP is the IP for which FIB is installed
2403             if (extIp.equals(dbExternalIp)) {
2404                 tempLabel = dbIpMap.getLabel();
2405                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2406                     tempLabel, dbExternalIp, routerId);
2407                 break;
2408             }
2409         }
2410         if (tempLabel == NatConstants.INVALID_ID) {
2411             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2412                     extIp, routerId);
2413             return;
2414         }
2415
2416         final long label = tempLabel;
2417         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2418         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2419                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2420                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2421         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2422
2423         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2424         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2425         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2426             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2427             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2428         }
2429         if (!switchOver) {
2430             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2431                     Futures.transformAsync(future, result -> {
2432                         //Release label
2433                         if (result.isSuccessful()) {
2434                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2435                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2436                             return vpnService.removeVpnLabel(labelInput);
2437                         } else {
2438                             String errMsg =
2439                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2440                                             + "prefix %s Failed - %s",
2441                                             dpnId, externalIp, result.getErrors());
2442                             LOG.error(errMsg);
2443                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2444                         }
2445                     }, MoreExecutors.directExecutor());
2446
2447             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2448
2449                 @Override
2450                 public void onFailure(@Nonnull Throwable error) {
2451                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2452                 }
2453
2454                 @Override
2455                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2456                     if (result.isSuccessful()) {
2457                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2458                             + "from VPN {}", externalIp, vpnName);
2459                     } else {
2460                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2461                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2462                     }
2463                 }
2464             }, MoreExecutors.directExecutor());
2465         } else {
2466             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2467                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2468         }
2469     }
2470
2471     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2472                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2473                                                TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2474             throws ExecutionException, InterruptedException {
2475         //Withdraw the corresponding routes from the BGP.
2476         //Get the network ID using the router ID.
2477         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2478                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2479         if (networkUuid == null) {
2480             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2481             return;
2482         }
2483
2484         if (externalIps == null || externalIps.isEmpty()) {
2485             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2486             return;
2487         }
2488
2489         if (vpnName == null) {
2490             //Get the VPN Name using the network ID
2491             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2492             if (vpnName == null) {
2493                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2494                     networkUuid, routerId);
2495                 return;
2496             }
2497         }
2498         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2499
2500         //Remove custom FIB routes
2501         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2502         for (String extIp : externalIps) {
2503             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2504                     writeFlowInvTx);
2505         }
2506     }
2507
2508     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2509         //Inform BGP about the route removal
2510         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2511         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2512         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2513     }
2514
2515     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2516             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2517         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2518         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2519             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2520         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2521     }
2522
2523     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2524             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2525         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2526         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2527         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2528         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2529     }
2530
2531     /**
2532      * router association to vpn.
2533      *
2534      * @param routerName - Name of router
2535      * @param routerId - router id
2536      * @param bgpVpnName BGP VPN name
2537      */
2538     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2539             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2540         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2541         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2542             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2543
2544             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2545
2546             if (bgpVpnId != NatConstants.INVALID_ID) {
2547                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2548                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2549                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2550                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2551                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2552                     getRoutersIdentifier(bgpVpnId), rtrs);
2553
2554                 // Get the allocated Primary NAPT Switch for this router
2555                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2556
2557                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2558                         routerId, bgpVpnId);
2559                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2560
2561                 // Get the group ID
2562                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2563                 createGroupId(getGroupIdKey(routerName));
2564                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2565                         extNwProvType);
2566             }
2567         }
2568     }
2569
2570     /**
2571      * router disassociation from vpn.
2572      *
2573      * @param routerName - Name of router
2574      * @param routerId - router id
2575      * @param bgpVpnName BGP VPN name
2576      */
2577     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2578             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2579         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2580         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2581             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2582             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2583
2584             // Get the allocated Primary NAPT Switch for this router
2585             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2586
2587             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2588             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2589
2590             // Get the group ID
2591             createGroupId(getGroupIdKey(routerName));
2592             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2593             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2594                     writeFlowInvTx, extNwProvType);
2595         }
2596     }
2597
2598     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2599         InstanceIdentifier<Routers> routerInstanceIndentifier =
2600             InstanceIdentifier.builder(ExtRouters.class)
2601                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2602         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2603         return routerData.isPresent() && routerData.get().isEnableSnat();
2604     }
2605
2606     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2607         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2608
2609         long changedVpnId = bgpVpnId;
2610         String idType = "BGP VPN";
2611         if (bgpVpnId == NatConstants.INVALID_ID) {
2612             changedVpnId = routerId;
2613             idType = "router";
2614         }
2615
2616         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2617         if (switches.isEmpty()) {
2618             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2619             return;
2620         }
2621         for (BigInteger dpnId : switches) {
2622             // Update the BGP VPN ID in the SNAT miss entry to group
2623             if (!dpnId.equals(primarySwitchId)) {
2624                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2625                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2626                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2627                 long groupId = createGroupId(getGroupIdKey(routerName));
2628                 if (!isSnatCfgd) {
2629                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2630                 }
2631
2632                 LOG.debug(
2633                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2634                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2635                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2636                 mdsalManager.addFlow(confTx, flowEntity);
2637             } else {
2638                 LOG.debug(
2639                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2640                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2641                 FlowEntity flowEntity =
2642                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2643                 mdsalManager.addFlow(confTx, flowEntity);
2644
2645                 LOG.debug(
2646                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2647                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2648                         idType, changedVpnId, primarySwitchId);
2649                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2650                         changedVpnId, confTx, extNwProvType);
2651
2652                 LOG.debug(
2653                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2654                                 + "which punts the packet to the controller in the Primary switch {}",
2655                         idType, changedVpnId, primarySwitchId);
2656                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2657
2658                 LOG.debug(
2659                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2660                                 + " outgoing packet to FIB Table in the Primary switch {}",
2661                         idType, changedVpnId, primarySwitchId);
2662                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2663
2664                 LOG.debug(
2665                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2666                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2667                                 + " {}", idType, changedVpnId, primarySwitchId);
2668                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2669
2670                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2671                         primarySwitchId);
2672                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2673                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2674                 if (vpnId != NatConstants.INVALID_ID) {
2675                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2676                 }
2677             }
2678         }
2679     }
2680
2681     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2682         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2683         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2684         if (ipPortMapping == null) {
2685             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2686             return;
2687         }
2688         // Get the External Gateway MAC Address
2689         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2690         if (extGwMacAddress != null) {
2691             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2692                     extGwMacAddress, routerId);
2693         } else {
2694             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2695                     routerId);
2696             return;
2697         }
2698         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2699         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2700             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2701             for (IpPortMap ipPortMap : ipPortMaps) {
2702                 String ipPortInternal = ipPortMap.getIpPortInternal();
2703                 String[] ipPortParts = ipPortInternal.split(":");
2704                 if (ipPortParts.length != 2) {
2705                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2706                     return;
2707                 }
2708                 String internalIp = ipPortParts[0];
2709                 String internalPort = ipPortParts[1];
2710                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2711                         internalIp, internalPort);
2712                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2713                 NAPTEntryEvent.Protocol protocol;
2714                 switch (protocolTypes) {
2715                     case TCP:
2716                         protocol = NAPTEntryEvent.Protocol.TCP;
2717                         break;
2718                     case UDP:
2719                         protocol = NAPTEntryEvent.Protocol.UDP;
2720                         break;
2721                     default:
2722                         protocol = NAPTEntryEvent.Protocol.TCP;
2723                 }
2724                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2725                 SessionAddress externalAddress =
2726                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2727                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2728                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2729                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2730                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2731                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2732             }
2733         }
2734     }
2735
2736     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2737                                                           long changedVpnId) {
2738
2739         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2740             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2741         List<MatchInfo> matches = new ArrayList<>();
2742         matches.add(MatchEthernetType.IPV4);
2743         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2744
2745         List<ActionInfo> actionsInfo = new ArrayList<>();
2746         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2747                 routerName);
2748         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2749         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2750                 actionsInfo);
2751         actionsInfo.add(new ActionGroup(groupId));
2752         List<InstructionInfo> instructions = new ArrayList<>();
2753         instructions.add(new InstructionApplyActions(actionsInfo));
2754         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2755         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2756             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2757             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2758
2759         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2760         return flowEntity;
2761     }
2762
2763     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2764                                                                         long changedVpnId) {
2765
2766         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2767                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2768         List<MatchInfo> matches = new ArrayList<>();
2769         matches.add(MatchEthernetType.IPV4);
2770         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2771
2772         List<InstructionInfo> instructions = new ArrayList<>();
2773         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2774
2775         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2776         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2777             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2778             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2779
2780         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2781         return flowEntity;
2782     }
2783
2784     // TODO : Replace this with ITM Rpc once its available with full functionality
2785     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2786         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2787
2788         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2789             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2790         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2791                 extNwProvType);
2792         mdsalManager.addFlow(confTx, flowEntity);
2793     }
2794
2795     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2796             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2797         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2798             dpId, routerName, changedVpnId);
2799         List<MatchInfo> matches = new ArrayList<>();
2800         matches.add(MatchEthernetType.IPV4);
2801
2802         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2803         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2804             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2805         }
2806         matches.add(new MatchTunnelId(tunnelId));
2807
2808         List<InstructionInfo> instructions = new ArrayList<>();
2809         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2810             MetaDataUtil.METADATA_MASK_VRFID));
2811         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2812         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2813         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2814         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2815             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2816             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2817         return flowEntity;
2818     }
2819
2820     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2821                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2822         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2823             dpnId, routerId, changedVpnId);
2824         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2825                 NwConstants.IP_PROT_TCP);
2826         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2827         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2828
2829         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2830                 NwConstants.IP_PROT_UDP);
2831         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2832         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2833
2834         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2835         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2836         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2837     }
2838
2839     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2840                                                            long changedVpnId, int protocol) {
2841         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2842             dpId, routerId, changedVpnId);
2843         BigInteger cookie = getCookieOutboundFlow(routerId);
2844         List<MatchInfo> matches = new ArrayList<>();
2845         matches.add(MatchEthernetType.IPV4);
2846         matches.add(new MatchIpProtocol((short)protocol));
2847         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2848
2849         List<InstructionInfo> instructions = new ArrayList<>();
2850         List<ActionInfo> actionsInfos = new ArrayList<>();
2851         actionsInfos.add(new ActionPuntToController());
2852         if (snatPuntTimeout != 0) {
2853             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2854         }
2855         instructions.add(new InstructionApplyActions(actionsInfos));
2856
2857         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2858         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2859             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2860         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2861         return flowEntity;
2862     }
2863
2864     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2865                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2866         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2867             dpnId, segmentId, changedVpnId);
2868         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2869         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2870     }
2871
2872     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2873
2874         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2875             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2876         List<MatchInfo> matches = new ArrayList<>();
2877         matches.add(MatchEthernetType.IPV4);
2878         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2879
2880         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2881         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2882         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2883         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2884         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2885
2886         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2887         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2888             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2889             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2890         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2891         return flowEntity;
2892     }
2893
2894     @Override
2895     protected ExternalRoutersListener getDataTreeChangeListener() {
2896         return ExternalRoutersListener.this;
2897     }
2898
2899     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2900                                                             TypedWriteTransaction<Configuration> writeFlowInvTx) {
2901         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2902                 routerName);
2903         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2904             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2905             if (subnetVpnId != -1) {
2906                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2907                     + "and vpnId {}", dpnId, subnetVpnId);
2908                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2909             }
2910         }
2911     }
2912 }