natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.FutureCallback;
  12 import com.google.common.util.concurrent.Futures;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14 import com.google.common.util.concurrent.MoreExecutors;
  15 import java.math.BigInteger;
  16 import java.net.Inet6Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Set;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import javax.annotation.Nonnull;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  36 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.mdsalutil.ActionInfo;
  43 import org.opendaylight.genius.mdsalutil.BucketInfo;
  44 import org.opendaylight.genius.mdsalutil.FlowEntity;
  45 import org.opendaylight.genius.mdsalutil.GroupEntity;
  46 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.MatchInfo;
  49 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  50 import org.opendaylight.genius.mdsalutil.NwConstants;
  51 import org.opendaylight.genius.mdsalutil.UpgradeState;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  58 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  60 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  61 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  66 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  67 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  68 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  69 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  70 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  71 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  72 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  73 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  74 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 142 import org.opendaylight.yangtools.yang.binding.DataObject;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.opendaylight.yangtools.yang.common.RpcResult;
 145 import org.slf4j.Logger;
 146 import org.slf4j.LoggerFactory;
 147
 148 @Singleton
 149 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 150     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 151
 152     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 153     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 154
 155     private final DataBroker dataBroker;
 156     private final ManagedNewTransactionRunner txRunner;
 157     private final IMdsalApiManager mdsalManager;
 158     private final ItmRpcService itmManager;
 159     private final OdlInterfaceRpcService odlInterfaceRpcService;
 160     private final IdManagerService idManager;
 161     private final NaptManager naptManager;
 162     private final NAPTSwitchSelector naptSwitchSelector;
 163     private final IBgpManager bgpManager;
 164     private final VpnRpcService vpnService;
 165     private final FibRpcService fibService;
 166     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 167     private final NaptEventHandler naptEventHandler;
 168     private final NaptPacketInHandler naptPacketInHandler;
 169     private final IFibManager fibManager;
 170     private final IVpnManager vpnManager;
 171     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 172     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 173     private final NatMode natMode;
 174     private final INeutronVpnManager nvpnManager;
 175     private final IElanService elanManager;
 176     private final JobCoordinator coordinator;
 177     private final UpgradeState upgradeState;
 178     private final IInterfaceManager interfaceManager;
 179
 180     @Inject
 181     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 182                                    final ItmRpcService itmManager,
 183                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 184                                    final IdManagerService idManager,
 185                                    final NaptManager naptManager,
 186                                    final NAPTSwitchSelector naptSwitchSelector,
 187                                    final IBgpManager bgpManager,
 188                                    final VpnRpcService vpnService,
 189                                    final FibRpcService fibService,
 190                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 191                                    final NaptEventHandler naptEventHandler,
 192                                    final NaptPacketInHandler naptPacketInHandler,
 193                                    final IFibManager fibManager,
 194                                    final IVpnManager vpnManager,
 195                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 196                                    final INeutronVpnManager nvpnManager,
 197                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 198                                    final NatserviceConfig config,
 199                                    final IElanService elanManager,
 200                                    final JobCoordinator coordinator,
 201                                    final UpgradeState upgradeState,
 202                                    final IInterfaceManager interfaceManager) {
 203         super(Routers.class, ExternalRoutersListener.class);
 204         this.dataBroker = dataBroker;
 205         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 206         this.mdsalManager = mdsalManager;
 207         this.itmManager = itmManager;
 208         this.odlInterfaceRpcService = odlInterfaceRpcService;
 209         this.idManager = idManager;
 210         this.naptManager = naptManager;
 211         this.naptSwitchSelector = naptSwitchSelector;
 212         this.bgpManager = bgpManager;
 213         this.vpnService = vpnService;
 214         this.fibService = fibService;
 215         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 216         this.naptEventHandler = naptEventHandler;
 217         this.naptPacketInHandler = naptPacketInHandler;
 218         this.fibManager = fibManager;
 219         this.vpnManager = vpnManager;
 220         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 221         this.nvpnManager = nvpnManager;
 222         this.elanManager = elanManager;
 223         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 224         this.coordinator = coordinator;
 225         this.upgradeState = upgradeState;
 226         this.interfaceManager = interfaceManager;
 227         if (config != null) {
 228             this.natMode = config.getNatMode();
 229         } else {
 230             this.natMode = NatMode.Controller;
 231         }
 232     }
 233
 234     @Override
 235     @PostConstruct
 236     public void init() {
 237         LOG.info("{} init", getClass().getSimpleName());
 238         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 239         createGroupIdPool();
 240     }
 241
 242     @Override
 243     protected InstanceIdentifier<Routers> getWildCardPath() {
 244         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 245     }
 246
 247     @Override
 248     // TODO Clean up the exception handling
 249     @SuppressWarnings("checkstyle:IllegalCatch")
 250     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 251         // Populate the router-id-name container
 252         String routerName = routers.getRouterName();
 253         LOG.info("add : external router event for {}", routerName);
 254         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 255         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 256         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 257         if (natMode == NatMode.Conntrack && !upgradeState.isUpgradeInProgress()) {
 258             if (bgpVpnUuid != null) {
 259                 return;
 260             }
 261             List<ExternalIps> externalIps = routers.getExternalIps();
 262             // Allocate Primary Napt Switch for this router
 263             if (routers.isEnableSnat() && externalIps != null && !externalIps.isEmpty()) {
 264                 centralizedSwitchScheduler.scheduleCentralizedSwitch(routers);
 265             }
 266             //snatServiceManger.notify(routers, null, Action.ADD);
 267         } else {
 268             try {
 269                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.getKey(),
 270                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 271                         LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 272                         long bgpVpnId = NatConstants.INVALID_ID;
 273                         if (bgpVpnUuid != null) {
 274                             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 275                         }
 276                         addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, tx);
 277                         // Allocate Primary Napt Switch for this router
 278                         BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 279                         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 280                             if (!routers.isEnableSnat()) {
 281                                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 282                                 /* If SNAT is disabled on ext-router though L3_FIB_TABLE(21) -> PSNAT_TABLE(26) flow
 283                                  * is required for DNAT. Hence writeFlowInvTx object submit is required.
 284                                  */
 285                                 return;
 286                             }
 287                             handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, tx);
 288                         }
 289                     })), NatConstants.NAT_DJC_MAX_RETRIES);
 290             } catch (Exception ex) {
 291                 LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 292                         routerName, ex);
 293             }
 294         }
 295     }
 296
 297     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 298                                  WriteTransaction writeFlowInvTx) {
 299         String routerName = routers.getRouterName();
 300         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 301
 302         naptManager.initialiseExternalCounter(routers, routerId);
 303         subnetRegisterMapping(routers, routerId);
 304
 305         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 306             primarySwitchId, routerName);
 307
 308         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 309                 routers.getNetworkId());
 310         if (extNwProvType == null) {
 311             LOG.error("handleEnableSnat : External Network Provider Type missing");
 312             return;
 313         }
 314
 315         if (bgpVpnId != NatConstants.INVALID_ID) {
 316             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, writeFlowInvTx,
 317                     extNwProvType);
 318         } else {
 319             // write metadata and punt
 320             installOutboundMissEntry(routerName, routerId, primarySwitchId, writeFlowInvTx);
 321             // Now install entries in SNAT tables to point to Primary for each router
 322             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 323             for (BigInteger dpnId : switches) {
 324                 // Handle switches and NAPT switches separately
 325                 if (!dpnId.equals(primarySwitchId)) {
 326                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 327                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 328                 } else {
 329                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 330                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, writeFlowInvTx);
 331                 }
 332             }
 333         }
 334
 335         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 336         if (externalIps.isEmpty()) {
 337             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 338             return;
 339         } else {
 340             for (String externalIpAddrPrefix : externalIps) {
 341                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 342                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 343                 handleSnatReverseTraffic(primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix,
 344                         writeFlowInvTx);
 345             }
 346         }
 347         LOG.debug("handleEnableSnat : Exit");
 348     }
 349
 350     private BigInteger getPrimaryNaptSwitch(String routerName) {
 351         // Allocate Primary Napt Switch for this router
 352         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 353         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 354             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 355                 primarySwitchId, routerName);
 356             return primarySwitchId;
 357         }
 358         // Validating and creating VNI pool during when NAPT switch is selected.
 359         // With Assumption this might be the first NAT service comes up.
 360         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 361                 NatConstants.ODL_VNI_POOL_NAME);
 362         // Allocated an id from VNI pool for the Router.
 363         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 364         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 365         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 366
 367         return primarySwitchId;
 368     }
 369
 370     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 371                                                      WriteTransaction writeFlowInvTx) {
 372         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 373         if (extVpnId == NatConstants.INVALID_ID) {
 374             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 375             extVpnId = routerId;
 376         }
 377         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 378         if (externalIps.isEmpty()) {
 379             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 380                 dpnId, extVpnId);
 381             return;
 382         }
 383         for (String ip : externalIps) {
 384             Uuid subnetId = getSubnetIdForFixedIp(ip);
 385             if (subnetId != null) {
 386                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 387                 if (subnetVpnId != NatConstants.INVALID_ID) {
 388                     extVpnId = subnetVpnId;
 389                 }
 390                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 391                     dpnId, extVpnId, subnetId);
 392                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 393                 mdsalManager.addFlowToTx(postNaptFlowEntity, writeFlowInvTx);
 394             }
 395         }
 396     }
 397
 398     private Uuid getSubnetIdForFixedIp(String ip) {
 399         if (ip != null) {
 400             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 401             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 402             Uuid subnetId = NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 403             return subnetId;
 404         }
 405         LOG.error("getSubnetIdForFixedIp : ip is null");
 406         return null;
 407     }
 408
 409     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 410         List<Uuid> subnetList = null;
 411         List<String> externalIps = null;
 412         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 413         subnetList = routerEntry.getSubnetIds();
 414         externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 415         int counter = 0;
 416         int extIpCounter = externalIps.size();
 417         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 418                 counter, extIpCounter);
 419         for (Uuid subnet : subnetList) {
 420             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 421             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 422                 .builder(Subnetmaps.class)
 423                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 424                 .build();
 425             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 426             if (sn.isPresent()) {
 427                 // subnets
 428                 Subnetmap subnetmapEntry = sn.get();
 429                 String subnetString = subnetmapEntry.getSubnetIp();
 430                 String[] subnetSplit = subnetString.split("/");
 431                 String subnetIp = subnetSplit[0];
 432                 try {
 433                     InetAddress address = InetAddress.getByName(subnetIp);
 434                     if (address instanceof Inet6Address) {
 435                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 436                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 437                         continue;
 438                     }
 439                 } catch (UnknownHostException e) {
 440                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 441                     return;
 442                 }
 443                 String subnetPrefix = "0";
 444                 if (subnetSplit.length == 2) {
 445                     subnetPrefix = subnetSplit[1];
 446                 }
 447                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 448                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 449                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 450                 //externalIps
 451                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 452                         counter, extIpCounter);
 453                 if (extIpCounter != 0) {
 454                     if (counter < extIpCounter) {
 455                         String[] ipSplit = externalIps.get(counter).split("/");
 456                         String externalIp = ipSplit[0];
 457                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 458                         if (ipSplit.length == 2) {
 459                             extPrefix = ipSplit[1];
 460                         }
 461                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 462                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 463                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 464                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 465                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 466                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 467                     } else {
 468                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 469                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 470                         String[] ipSplit = externalIps.get(counter).split("/");
 471                         String externalIp = ipSplit[0];
 472                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 473                         if (ipSplit.length == 2) {
 474                             extPrefix = ipSplit[1];
 475                         }
 476                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 477                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 478                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 479                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 480                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 481                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 482                             externalIp, extPrefix);
 483                     }
 484                 }
 485                 counter++;
 486                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 487             } else {
 488                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 489             }
 490         }
 491     }
 492
 493     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 494             Uuid bgpVpnUuid, boolean create, WriteTransaction writeFlowInvTx) {
 495         //Check if BGP VPN exists. If exists then invoke the new method.
 496         if (bgpVpnId != NatConstants.INVALID_ID) {
 497             if (bgpVpnUuid != null) {
 498                 String bgpVpnName = bgpVpnUuid.getValue();
 499                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 500                     bgpVpnId, bgpVpnName);
 501                 RouterIds rtrs = new RouterIdsBuilder().setKey(new RouterIdsKey(bgpVpnId))
 502                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 503                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 504                     getRoutersIdentifier(bgpVpnId), rtrs);
 505             }
 506             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, create, writeFlowInvTx);
 507             return;
 508         }
 509
 510         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 511         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, writeFlowInvTx);
 512     }
 513
 514     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 515             WriteTransaction writeFlowInvTx) {
 516         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 517         if (switches.isEmpty()) {
 518             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 519             return;
 520         }
 521         if (routerId == NatConstants.INVALID_ID) {
 522             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 523                     + "default NAT route in FIB", routerName);
 524             return;
 525         }
 526         for (BigInteger dpnId : switches) {
 527             if (create) {
 528                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 529                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 530                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 531             } else {
 532                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 533                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 534                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 535             }
 536         }
 537     }
 538
 539     private void addOrDelDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 540             long bgpVpnId, boolean create,WriteTransaction writeFlowInvTx) {
 541         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 542         if (dpnIds.isEmpty()) {
 543             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 544                     + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 545             return;
 546         }
 547         for (BigInteger dpnId : dpnIds) {
 548             if (create) {
 549                 if (bgpVpnId != NatConstants.INVALID_ID) {
 550                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 551                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 552                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 553                 } else {
 554                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 555                             + "in dpn {} for the internal vpn", routerId, dpnId);
 556                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId,writeFlowInvTx);
 557                 }
 558             } else {
 559                 if (bgpVpnId != NatConstants.INVALID_ID) {
 560                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 561                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 562                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 563                 } else {
 564                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 565                             + "in dpn {} for the internal vpn", routerId, dpnId);
 566                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 567                 }
 568             }
 569         }
 570     }
 571
 572     // TODO Clean up the exception handling
 573     @SuppressWarnings("checkstyle:IllegalCatch")
 574     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 575                                                           InstanceIdentifier<T> path) {
 576         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 577
 578         try {
 579             return tx.read(datastoreType, path).get();
 580         } catch (Exception e) {
 581             throw new RuntimeException(e);
 582         }
 583     }
 584
 585     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 586                                             WriteTransaction writeFlowInvTx) {
 587         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 588         if (routerId != NatConstants.INVALID_ID) {
 589             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 590                     primarySwitchId, routerId);
 591             createOutboundTblEntry(primarySwitchId, routerId, writeFlowInvTx);
 592         } else {
 593             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 594                 + "createAndInstallMissEntry", routerName);
 595         }
 596     }
 597
 598     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID) {
 599         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 600                 .FLOWID_SEPARATOR + routerID;
 601     }
 602
 603     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 604         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 605                 .FLOWID_SEPARATOR + vpnId;
 606     }
 607
 608     public BigInteger getCookieOutboundFlow(long routerId) {
 609         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 610             BigInteger.valueOf(routerId));
 611     }
 612
 613     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId) {
 614         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 615         List<MatchInfo> matches = new ArrayList<>();
 616         matches.add(MatchEthernetType.IPV4);
 617         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 618
 619         List<InstructionInfo> instructions = new ArrayList<>();
 620         List<ActionInfo> actionsInfos = new ArrayList<>();
 621         actionsInfos.add(new ActionPuntToController());
 622         instructions.add(new InstructionApplyActions(actionsInfos));
 623         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 624             MetaDataUtil.METADATA_MASK_VRFID));
 625
 626         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
 627         BigInteger cookie = getCookieOutboundFlow(routerId);
 628         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 629             5, flowRef, 0, 0,
 630             cookie, matches, instructions);
 631         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 632         return flowEntity;
 633     }
 634
 635     public void createOutboundTblEntry(BigInteger dpnId, long routerId, WriteTransaction writeFlowInvTx) {
 636         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 637         FlowEntity flowEntity = buildOutboundFlowEntity(dpnId, routerId);
 638         LOG.debug("createOutboundTblEntry : Installing flow {}", flowEntity);
 639         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 640     }
 641
 642     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 643         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 644         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 645         try {
 646             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 647                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 648                     .setSourceDpid(srcDpId)
 649                     .setDestinationDpid(dstDpId)
 650                     .setTunnelType(tunType)
 651                     .build());
 652             rpcResult = result.get();
 653             if (!rpcResult.isSuccessful()) {
 654                 tunType = TunnelTypeGre.class;
 655                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 656                     .setSourceDpid(srcDpId)
 657                     .setDestinationDpid(dstDpId)
 658                     .setTunnelType(tunType)
 659                     .build());
 660                 rpcResult = result.get();
 661                 if (!rpcResult.isSuccessful()) {
 662                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 663                             rpcResult.getErrors());
 664                 } else {
 665                     return rpcResult.getResult().getInterfaceName();
 666                 }
 667                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 668                         rpcResult.getErrors());
 669             } else {
 670                 return rpcResult.getResult().getInterfaceName();
 671             }
 672         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 673             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 674                     + "between {} and {}", srcDpId, dstDpId, e);
 675         }
 676
 677         return null;
 678     }
 679
 680     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 681                                                     WriteTransaction writeFlowInvTx) {
 682         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 683         // Install miss entry pointing to group
 684         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 685         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 686     }
 687
 688     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 689             String routerName, long routerId) {
 690         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 691             dpnId, bucketInfo.get(0));
 692         // Install the select group
 693         long groupId = createGroupId(getGroupIdKey(routerName));
 694         GroupEntity groupEntity =
 695             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 696         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 697         mdsalManager.syncInstallGroup(groupEntity);
 698         // Install miss entry pointing to group
 699         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 700         if (flowEntity == null) {
 701             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 702                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 703                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 704             return;
 705         }
 706         mdsalManager.installFlow(flowEntity);
 707     }
 708
 709     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 710         long groupId = createGroupId(getGroupIdKey(routerName));
 711         GroupEntity groupEntity =
 712             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 713         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 714         mdsalManager.syncInstallGroup(groupEntity);
 715         return groupId;
 716     }
 717
 718     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 719         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 720                 dpId, routerName, groupId);
 721         List<MatchInfo> matches = new ArrayList<>();
 722         matches.add(MatchEthernetType.IPV4);
 723         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 724
 725         List<ActionInfo> actionsInfo = new ArrayList<>();
 726         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 727                 routerName);
 728         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 729         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 730         actionsInfo.add(new ActionGroup(groupId));
 731         List<InstructionInfo> instructions = new ArrayList<>();
 732         instructions.add(new InstructionApplyActions(actionsInfo));
 733         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 734         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 735                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 736                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 737
 738         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 739         return flowEntity;
 740     }
 741
 742     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 743
 744         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 745             routerName);
 746         List<MatchInfo> matches = new ArrayList<>();
 747         matches.add(MatchEthernetType.IPV4);
 748         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 749
 750         List<InstructionInfo> instructions = new ArrayList<>();
 751         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 752
 753         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 754         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 755             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 756             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 757
 758         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 759         return flowEntity;
 760     }
 761
 762     // TODO : Replace this with ITM Rpc once its available with full functionality
 763     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 764                                                      WriteTransaction writeFlowInvTx) {
 765         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 766             dpnId, routerName);
 767         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 768         if (flowEntity == null) {
 769             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 770                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 771                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 772                     dpnId, routerName);
 773             return;
 774         }
 775         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 776
 777     }
 778
 779     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 780         List<MatchInfo> matches = new ArrayList<>();
 781         matches.add(MatchEthernetType.IPV4);
 782         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 783                 routerName);
 784         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 785         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 786         List<InstructionInfo> instructions = new ArrayList<>();
 787         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 788                 MetaDataUtil.METADATA_MASK_VRFID));
 789         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 790         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 791                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 792                 instructions);
 793         return flowEntity;
 794     }
 795
 796     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 797         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 798                 .FLOWID_SEPARATOR + routerID;
 799     }
 800
 801     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 802         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 803                 .FLOWID_SEPARATOR + routerID;
 804     }
 805
 806     private String getGroupIdKey(String routerName) {
 807         return "snatmiss." + routerName;
 808     }
 809
 810     protected long createGroupId(String groupIdKey) {
 811         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 812             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 813             .build();
 814         try {
 815             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 816             RpcResult<AllocateIdOutput> rpcResult = result.get();
 817             return rpcResult.getResult().getIdValue();
 818         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 819             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 820         }
 821         return 0;
 822     }
 823
 824     protected void createGroupIdPool() {
 825         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 826             .setPoolName(NatConstants.SNAT_IDPOOL_NAME)
 827             .setLow(NatConstants.SNAT_ID_LOW_VALUE)
 828             .setHigh(NatConstants.SNAT_ID_HIGH_VALUE)
 829             .build();
 830         try {
 831             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 832             if (result != null && result.get().isSuccessful()) {
 833                 LOG.debug("createGroupIdPool : GroupIdPool created successfully");
 834             } else {
 835                 LOG.error("createGroupIdPool : Unable to create GroupIdPool");
 836             }
 837         } catch (InterruptedException | ExecutionException e) {
 838             LOG.error("createGroupIdPool : Failed to create PortPool for NAPT Service", e);
 839         }
 840     }
 841
 842     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 843         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 844         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 845         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 846         List<BucketInfo> listBucketInfo = new ArrayList<>();
 847
 848         if (ifNamePrimary != null) {
 849             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 850             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 851                     interfaceManager, ifNamePrimary, routerId);
 852             if (listActionInfoPrimary.isEmpty()) {
 853                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 854                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 855                         ifNamePrimary);
 856             }
 857         } else {
 858             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 859                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 860         }
 861         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 862
 863         listBucketInfo.add(0, bucketPrimary);
 864         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 865     }
 866
 867     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 868                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 869         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 870         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 871         List<BucketInfo> listBucketInfo = new ArrayList<>();
 872
 873         if (ifNamePrimary != null) {
 874             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 875                     ifNamePrimary);
 876             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 877                     interfaceManager, ifNamePrimary, routerId);
 878             if (listActionInfoPrimary.isEmpty()) {
 879                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 880                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 881                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 882             }
 883         } else {
 884             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 885                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 886         }
 887         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 888
 889         listBucketInfo.add(0, bucketPrimary);
 890         return listBucketInfo;
 891     }
 892
 893     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 894             WriteTransaction writeFlowInvTx) {
 895        /*
 896         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 897         */
 898
 899         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 900
 901 /*
 902         List<BucketInfo> listBucketInfo = new ArrayList<>();
 903         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 904         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 905         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 906         listBucketInfo.add(0, bucketPrimary);
 907 */
 908
 909         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, writeFlowInvTx);
 910         installTerminatingServiceTblEntry(dpnId, routerName, routerId, writeFlowInvTx);
 911         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 912         installNaptPfibEntry(dpnId, routerId, writeFlowInvTx);
 913         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 914         if (networkId != null) {
 915             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 916             if (vpnUuid != null) {
 917                 Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 918                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 919                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 920                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 921                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 922                         installNaptPfibEntry(dpnId, vpnId, null);
 923                     }
 924                     return Collections.emptyList();
 925                 });
 926             } else {
 927                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 928             }
 929         } else {
 930             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 931         }
 932     }
 933
 934     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 935         List<BucketInfo> listBucketInfo = new ArrayList<>();
 936         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 937         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 938         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 939         listBucketInfo.add(0, bucketPrimary);
 940         return listBucketInfo;
 941     }
 942
 943     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, WriteTransaction writeFlowInvTx) {
 944         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 945         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 946         if (writeFlowInvTx != null) {
 947             mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
 948         } else {
 949             mdsalManager.installFlow(naptPfibFlowEntity);
 950         }
 951     }
 952
 953     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 954
 955         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 956         List<MatchInfo> matches = new ArrayList<>();
 957         matches.add(MatchEthernetType.IPV4);
 958         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 959
 960         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 961         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 962         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
 963         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 964         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 965
 966         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
 967         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
 968             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 969             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
 970         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
 971         return flowEntity;
 972     }
 973
 974     public void handleSnatReverseTraffic(BigInteger dpnId, Routers router, long routerId, String routerName,
 975             String externalIp, WriteTransaction writeFlowInvTx) {
 976         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
 977             dpnId, routerId, externalIp);
 978         Uuid networkId = router.getNetworkId();
 979         if (networkId == null) {
 980             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
 981             return;
 982         }
 983         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
 984         if (vpnName == null) {
 985             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
 986                     + "configuration {} in router {}", networkId, externalIp, routerId);
 987             return;
 988         }
 989         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
 990             externalIp, networkId, router, writeFlowInvTx);
 991         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
 992             dpnId, routerId, externalIp);
 993     }
 994
 995     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
 996                                                 final long routerId, final String routerName, final String externalIp,
 997                                                 final Uuid extNetworkId, final Routers router,
 998                                                 final WriteTransaction writeFlowInvTx) {
 999         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1000                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1001         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1002         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1003         if (rd == null || rd.isEmpty()) {
1004             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1005             return;
1006         }
1007         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1008         if (extNwProvType == null) {
1009             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1010             return;
1011         }
1012         if (extNwProvType == ProviderTypes.VXLAN) {
1013             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1014                 tx -> evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp,
1015                         vpnName, rd, nextHopIp, tx, routerId, routerName, writeFlowInvTx)), LOG,
1016                 "Error installing FIB and TS flows");
1017             return;
1018         }
1019         //Generate VPN label for the external IP
1020         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1021             .setIpPrefix(externalIp).build();
1022         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1023
1024         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1025         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1026             if (result.isSuccessful()) {
1027                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1028                 GenerateVpnLabelOutput output = result.getResult();
1029                 final long label = output.getLabel();
1030
1031                 int externalIpInDsFlag = 0;
1032                 //Get IPMaps from the DB for the router ID
1033                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1034                 if (dbIpMaps != null) {
1035                     for (IpMap dbIpMap : dbIpMaps) {
1036                         String dbExternalIp = dbIpMap.getExternalIp();
1037                         //Select the IPMap, whose external IP is the IP for which FIB is installed
1038                         if (dbExternalIp.contains(externalIp)) {
1039                             String dbInternalIp = dbIpMap.getInternalIp();
1040                             IpMapKey dbIpMapKey = dbIpMap.getKey();
1041                             LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1042                                     + "and externalIp {}", label, dbInternalIp, externalIp);
1043                             IpMap newIpm = new IpMapBuilder().setKey(dbIpMapKey).setInternalIp(dbInternalIp)
1044                                 .setExternalIp(dbExternalIp).setLabel(label).build();
1045                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1046                                 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1047                             externalIpInDsFlag++;
1048                         }
1049                     }
1050                     if (externalIpInDsFlag <= 0) {
1051                         LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1052                                 + "Failed to update label {} for routerId {} in DS",
1053                                 externalIp, label, routerId);
1054                         String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1055                             + " found in DS for router %s", label, externalIp, routerId);
1056                         return Futures.immediateFailedFuture(new Exception(errMsg));
1057                     }
1058                 } else {
1059                     LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1060                             + " routerId {} in DS", label, externalIp, routerId);
1061                 }
1062                 //Inform BGP
1063                 long l3vni = 0;
1064                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1065                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1066                 }
1067                 Routers extRouter = router != null ? router :
1068                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1069                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1070                         extRouter);
1071                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1072                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1073                     RouteOrigin.STATIC, dpnId);
1074
1075                 //Install custom FIB routes
1076                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1077                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1078                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, writeFlowInvTx,
1079                         extNwProvType);
1080                 makeLFibTableEntry(dpnId, label, tableId, writeFlowInvTx);
1081
1082                 //Install custom FIB routes - FIB table.
1083                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1084                         routerName, externalIp);
1085                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1086                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1087                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId,
1088                             NwConstants.INBOUND_NAPT_TABLE,writeFlowInvTx);
1089                 }
1090                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1091                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1092                         externalSubnetId);
1093                 String externalVpn = vpnName;
1094                 if (externalSubnet.isPresent()) {
1095                     externalVpn =  externalSubnetId.getValue();
1096                 }
1097                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1098                     .setVpnName(externalVpn)
1099                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1100                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1101                     .setInstruction(fibTableCustomInstructions).build();
1102                 return fibService.createFibEntry(input);
1103             } else {
1104                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1105                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1106                     externalIp, vpnName, result.getErrors());
1107                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1108             }
1109         }, MoreExecutors.directExecutor());
1110
1111         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1112
1113             @Override
1114             public void onFailure(@Nonnull Throwable error) {
1115                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1116             }
1117
1118             @Override
1119             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1120                 if (result.isSuccessful()) {
1121                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1122                             externalIp);
1123                 } else {
1124                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1125                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1126                 }
1127             }
1128         }, MoreExecutors.directExecutor());
1129     }
1130
1131     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1132             String externalIp) {
1133         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1134         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1135         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1136                 externalIp, router);
1137         int instructionIndex = 0;
1138         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1139             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1140             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1141                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1142             instructionIndex++;
1143         }
1144
1145         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1146         return fibTableCustomInstructions;
1147     }
1148
1149     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId, WriteTransaction writeFlowInvTx) {
1150         List<MatchInfo> matches = new ArrayList<>();
1151         matches.add(MatchEthernetType.MPLS_UNICAST);
1152         matches.add(new MatchMplsLabel(serviceId));
1153
1154         List<Instruction> instructions = new ArrayList<>();
1155         List<ActionInfo> actionsInfos = new ArrayList<>();
1156         actionsInfos.add(new ActionPopMpls());
1157         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1158         instructions.add(writeInstruction);
1159         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1160
1161         // Install the flow entry in L3_LFIB_TABLE
1162         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1163
1164         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1165             10, flowRef, 0, 0,
1166             COOKIE_VM_LFIB_TABLE, matches, instructions);
1167
1168         mdsalManager.addFlowToTx(dpId, flowEntity, writeFlowInvTx);
1169
1170         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1171     }
1172
1173     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1174              List<Instruction> customInstructions, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
1175         List<MatchInfo> mkMatches = new ArrayList<>();
1176
1177         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1178             dpnId, serviceId, customInstructions);
1179
1180         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1181             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1182         } else {
1183             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1184         }
1185
1186         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1187             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1188             String.format("%s:%d", "TST Flow Entry ", serviceId),
1189             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1190
1191         mdsalManager.addFlowToTx(dpnId, terminatingServiceTableFlowEntity, writeFlowInvTx);
1192     }
1193
1194     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1195         InstanceIdentifier<RouterIds> id = InstanceIdentifier.builder(
1196             RouterIdName.class).child(RouterIds.class, new RouterIdsKey(routerId)).build();
1197         return id;
1198     }
1199
1200     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1201         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1202                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1203     }
1204
1205     @Override
1206     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1207         String routerName = original.getRouterName();
1208         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1209         if (routerId == NatConstants.INVALID_ID) {
1210             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1211             return;
1212         }
1213         // Check if its update on SNAT flag
1214         boolean originalSNATEnabled = original.isEnableSnat();
1215         boolean updatedSNATEnabled = update.isEnableSnat();
1216         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1217             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1218         if (natMode == NatMode.Conntrack && !upgradeState.isUpgradeInProgress()) {
1219             if (originalSNATEnabled != updatedSNATEnabled) {
1220                 BigInteger primarySwitchId;
1221                 if (originalSNATEnabled) {
1222                     //SNAT disabled for the router
1223                     centralizedSwitchScheduler.releaseCentralizedSwitch(update);
1224                 } else {
1225                     centralizedSwitchScheduler.scheduleCentralizedSwitch(update);
1226                 }
1227             } else if (updatedSNATEnabled) {
1228                 centralizedSwitchScheduler.updateCentralizedSwitch(original,update);
1229             }
1230             List<ExternalIps> originalExternalIps = original.getExternalIps();
1231             List<ExternalIps> updateExternalIps = update.getExternalIps();
1232             if (!Objects.equals(originalExternalIps, updateExternalIps)) {
1233                 if (originalExternalIps == null || originalExternalIps.isEmpty()) {
1234                     centralizedSwitchScheduler.scheduleCentralizedSwitch(update);
1235                 }
1236             }
1237         } else {
1238             /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1239              * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1240              */
1241             long bgpVpnId = NatConstants.INVALID_ID;
1242             Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1243             if (bgpVpnUuid != null) {
1244                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1245             }
1246             BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1247             if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1248                 // Router has no interface attached
1249                 return;
1250             }
1251             final long finalBgpVpnId = bgpVpnId;
1252             coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.getKey(), () -> {
1253                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1254                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeFlowInvTx -> {
1255                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(removeFlowInvTx -> {
1256                         Uuid networkId = original.getNetworkId();
1257                         if (originalSNATEnabled != updatedSNATEnabled) {
1258                             if (originalSNATEnabled) {
1259                                 //SNAT disabled for the router
1260                                 Uuid networkUuid = original.getNetworkId();
1261                                 LOG.info("update : SNAT disabled for Router {}", routerName);
1262                                 Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1263                                 handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1264                                         removeFlowInvTx);
1265                             } else {
1266                                 LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1267                                 handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1268                             }
1269                         }
1270                         if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1271                             NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1272                             NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1273                         }
1274
1275                         //Check if the Update is on External IPs
1276                         LOG.debug("update : Checking if this is update on External IPs");
1277                         List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1278                         List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1279
1280                         //Check if the External IPs are added during the update.
1281                         Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1282                         addedExternalIps.removeAll(originalExternalIps);
1283                         if (addedExternalIps.size() != 0) {
1284                             LOG.debug("update : Start processing of the External IPs addition during the update "
1285                                     + "operation");
1286                             vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1287                                     update.getExtGwMacAddress(), dpnId,
1288                                     update.getNetworkId(), null);
1289
1290                             for (String addedExternalIp : addedExternalIps) {
1291                     /*
1292                         1) Do nothing in the IntExtIp model.
1293                         2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1294                      */
1295                                 String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1296                                 String externalIp = externalIpParts[0];
1297                                 String externalIpPrefix = externalIpParts[1];
1298                                 String externalpStr = externalIp + "/" + externalIpPrefix;
1299                                 LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1300                                                 + "router ID {} in the ExternalIpsCounter model.",
1301                                         externalpStr, routerId);
1302                                 naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1303                             }
1304                             LOG.debug(
1305                                     "update : End processing of the External IPs addition during the update operation");
1306                         }
1307
1308                         //Check if the External IPs are removed during the update.
1309                         Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1310                         removedExternalIps.removeAll(updatedExternalIps);
1311                         if (removedExternalIps.size() > 0) {
1312                             LOG.debug("update : Start processing of the External IPs removal during the update "
1313                                     + "operation");
1314                             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1315                                     removedExternalIps, original.getExtGwMacAddress(),
1316                                     dpnId, networkId);
1317
1318                             for (String removedExternalIp : removedExternalIps) {
1319                     /*
1320                         1) Remove the mappings in the IntExt IP model which has external IP.
1321                         2) Remove the external IP in the ExternalCounter model.
1322                         3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1323                            least loaded external IP.
1324                            Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1325                         4) Increase the count of the allocated external IP by one.
1326                         5) Advertise to the BGP if external IP is allocated for the first time for the router
1327                          i.e. the route for the external IP is absent.
1328                         6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1329                          the removed external IPs and also from the model.
1330                         7) Advertise to the BGP for removing the route for the removed external IPs.
1331                      */
1332
1333                                 String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1334                                 String externalIp = externalIpParts[0];
1335                                 String externalIpPrefix = externalIpParts[1];
1336                                 String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1337
1338                                 LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1339                                         + "entries for removed external IP {}", externalIpAddrStr);
1340                                 Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1341                                 String vpnName = "";
1342                                 if (vpnUuId != null) {
1343                                     vpnName = vpnUuId.getValue();
1344                                 }
1345                                 clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1346                                         update.getExtGwMacAddress(), removeFlowInvTx);
1347
1348                                 LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1349                                 //Get the internal IPs which are associated to the removed external IPs
1350                                 List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1351                                 List<String> removedInternalIps = new ArrayList<>();
1352                                 for (IpMap ipMap : ipMaps) {
1353                                     if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1354                                         removedInternalIps.add(ipMap.getInternalIp());
1355                                     }
1356                                 }
1357
1358                                 LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1359                                 for (String removedInternalIp : removedInternalIps) {
1360                                     LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1361                                                     + "router ID {} from the IntExtIP model",
1362                                             removedInternalIp, routerId);
1363                                     naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1364                                 }
1365
1366                                 LOG.debug("update : Remove the count mapping of the external IP {} for the "
1367                                                 + "router ID {} from the ExternalIpsCounter model.",
1368                                         externalIpAddrStr, routerId);
1369                                 naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1370
1371                                 LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1372                                         + "whose external IPs were removed.");
1373                                 for (String removedInternalIp : removedInternalIps) {
1374                                     allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1375                                             removedInternalIp, writeFlowInvTx);
1376                                 }
1377
1378                                 LOG.debug("update : Remove the NAPT translation entries from "
1379                                         + "Inbound and Outbound NAPT tables for the removed external IPs.");
1380                                 //Get the internalIP and internal Port which were associated to the removed external IP.
1381                                 List<Integer> externalPorts = new ArrayList<>();
1382                                 Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1383                                 InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1384                                         .builder(IntextIpPortMap.class)
1385                                         .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1386                                 Optional<IpPortMapping> ipPortMapping =
1387                                         MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1388                                 if (ipPortMapping.isPresent()) {
1389                                     List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1390                                             .getIntextIpProtocolType();
1391                                     for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1392                                         ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1393                                         List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1394                                         for (IpPortMap ipPortMap : ipPortMaps) {
1395                                             IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1396                                             if (ipPortExternal.getIpAddress().equals(externalIp)) {
1397                                                 externalPorts.add(ipPortExternal.getPortNum());
1398                                                 List<String> removedInternalIpPorts =
1399                                                         protoTypesIntIpPortsMap.get(protoType);
1400                                                 if (removedInternalIpPorts != null) {
1401                                                     removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1402                                                     protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1403                                                 } else {
1404                                                     removedInternalIpPorts = new ArrayList<>();
1405                                                     removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1406                                                     protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1407                                                 }
1408                                             }
1409                                         }
1410                                     }
1411                                 }
1412
1413                                 //Remove the IP port map from the intext-ip-port-map model, which were containing
1414                                 // the removed external IP.
1415                                 Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1416                                         protoTypesIntIpPortsMap.entrySet();
1417                                 Map<String, List<String>> internalIpPortMap = new HashMap<>();
1418                                 for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1419                                     ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1420                                     List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1421                                     for (String removedInternalIpPort : removedInternalIpPorts) {
1422                                         // Remove the IP port map from the intext-ip-port-map model,
1423                                         // which were containing the removed external IP
1424                                         naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1425                                                 protocolType);
1426                                         //Remove the IP port incomint packer map.
1427                                         naptPacketInHandler.removeIncomingPacketMap(
1428                                                 routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1429                                         String[] removedInternalIpPortParts = removedInternalIpPort
1430                                                 .split(NatConstants.COLON_SEPARATOR);
1431                                         if (removedInternalIpPortParts.length == 2) {
1432                                             String removedInternalIp = removedInternalIpPortParts[0];
1433                                             String removedInternalPort = removedInternalIpPortParts[1];
1434                                             List<String> removedInternalPortsList =
1435                                                     internalIpPortMap.get(removedInternalPort);
1436                                             if (removedInternalPortsList != null) {
1437                                                 removedInternalPortsList.add(removedInternalPort);
1438                                                 internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1439                                             } else {
1440                                                 removedInternalPortsList = new ArrayList<>();
1441                                                 removedInternalPortsList.add(removedInternalPort);
1442                                                 internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1443                                             }
1444                                         }
1445                                     }
1446                                 }
1447
1448                                 // Delete the entry from SnatIntIpPortMap DS
1449                                 Set<String> internalIps = internalIpPortMap.keySet();
1450                                 for (String internalIp : internalIps) {
1451                                     LOG.debug("update : Removing IpPort having the internal IP {} from the "
1452                                             + "model SnatIntIpPortMap", internalIp);
1453                                     naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1454                                 }
1455
1456                                 naptManager.removeNaptPortPool(externalIp);
1457
1458                                 LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1459                                         + "the removed external IP {}", externalIp);
1460                                 for (Integer externalPort : externalPorts) {
1461                                     //Remove the NAPT translation entries from Inbound NAPT table
1462                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1463                                             routerId, externalIp, externalPort);
1464                                 }
1465
1466                                 Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1467                                 for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1468                                     String internalIp = internalIpPort.getKey();
1469                                     LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1470                                             + "for the removed internal IP {}", internalIp);
1471                                     List<String> internalPorts = internalIpPort.getValue();
1472                                     for (String internalPort : internalPorts) {
1473                                         //Remove the NAPT translation entries from Outbound NAPT table
1474                                         naptPacketInHandler.removeIncomingPacketMap(
1475                                                 routerId + NatConstants.COLON_SEPARATOR + internalIp
1476                                                         + NatConstants.COLON_SEPARATOR + internalPort);
1477                                         naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1478                                                 routerId, internalIp, Integer.parseInt(internalPort));
1479                                     }
1480                                 }
1481                             }
1482                             LOG.debug(
1483                                     "update : End processing of the External IPs removal during the update operation");
1484                         }
1485
1486                         //Check if its Update on subnets
1487                         LOG.debug("update : Checking if this is update on subnets");
1488                         List<Uuid> originalSubnetIds = original.getSubnetIds();
1489                         List<Uuid> updatedSubnetIds = update.getSubnetIds();
1490                         Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1491                         addedSubnetIds.removeAll(originalSubnetIds);
1492
1493                         //Check if the Subnet IDs are added during the update.
1494                         if (addedSubnetIds.size() != 0) {
1495                             LOG.debug(
1496                                     "update : Start processing of the Subnet IDs addition during the update operation");
1497                             for (Uuid addedSubnetId : addedSubnetIds) {
1498                     /*
1499                         1) Select the least loaded external IP for the subnet and store the mapping of the
1500                         subnet IP and the external IP in the IntExtIp model.
1501                         2) Increase the count of the selected external IP by one.
1502                         3) Advertise to the BGP if external IP is allocated for the first time for the
1503                         router i.e. the route for the external IP is absent.
1504                      */
1505                                 String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1506                                 if (subnetIp != null) {
1507                                     allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1508                                             writeFlowInvTx);
1509                                 }
1510                             }
1511                             LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1512                         }
1513
1514                         //Check if the Subnet IDs are removed during the update.
1515                         Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1516                         removedSubnetIds.removeAll(updatedSubnetIds);
1517                         if (removedSubnetIds.size() != 0) {
1518                             LOG.debug(
1519                                     "update : Start processing of the Subnet IDs removal during the update operation");
1520                             for (Uuid removedSubnetId : removedSubnetIds) {
1521                                 String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1522                                 if (subnetAddr != null) {
1523                         /*
1524                             1) Remove the subnet IP and the external IP in the IntExtIp map
1525                             2) Decrease the count of the coresponding external IP by one.
1526                             3) Advertise to the BGP for removing the routes of the corresponding external
1527                             IP if its not allocated to any other internal IP.
1528                         */
1529
1530                                     String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1531                                             subnetAddr[0] + "/" + subnetAddr[1]);
1532                                     if (externalIp == null) {
1533                                         LOG.error("update : No mapping found for router ID {} and internal IP {}",
1534                                                 routerId, subnetAddr[0]);
1535                                         return;
1536                                     }
1537
1538                                     naptManager.updateCounter(routerId, externalIp, false);
1539                                     // Traverse entire model of external-ip counter whether external ip is not
1540                                     // used by any other internal ip in any router
1541                                     if (!isExternalIpAllocated(externalIp)) {
1542                                         LOG.debug("update : external ip is not allocated to any other "
1543                                                 + "internal IP so proceeding to remove routes");
1544                                         clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1545                                                 Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1546                                                 removeFlowInvTx);
1547                                         LOG.debug("update : Successfully removed fib entries in switch {} for "
1548                                                         + "router {} with networkId {} and externalIp {}",
1549                                                 dpnId, routerId, networkId, externalIp);
1550                                     }
1551
1552                                     LOG.debug("update : Remove the IP mapping for the router ID {} and "
1553                                             + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1554                                     naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1555                                 }
1556                             }
1557                             LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1558                         }
1559                     }));
1560                 }));
1561                 return futures;
1562             }, NatConstants.NAT_DJC_MAX_RETRIES);
1563         } //end of controller based SNAT
1564     }
1565
1566     private boolean isExternalIpAllocated(String externalIp) {
1567         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1568         Optional<ExternalIpsCounter> externalCountersData =
1569             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1570         if (externalCountersData.isPresent()) {
1571             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1572             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1573             for (ExternalCounters ext : externalCounters) {
1574                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1575                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1576                         if (externalIpCount.getCounter() != 0) {
1577                             return true;
1578                         }
1579                         break;
1580                     }
1581                 }
1582             }
1583         }
1584         return false;
1585     }
1586
1587     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1588             Uuid networkId, String subnetIp, WriteTransaction writeFlowInvTx) {
1589         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1590         try {
1591             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1592             if (address instanceof Inet6Address) {
1593                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1594                 return;
1595             }
1596         } catch (UnknownHostException e) {
1597             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1598             return;
1599         }
1600         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1601         if (leastLoadedExtIpAddr != null) {
1602             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1603             String leastLoadedExtIp = externalIpParts[0];
1604             String leastLoadedExtIpPrefix = externalIpParts[1];
1605             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1606             subnetIp = subnetIpParts[0];
1607             String subnetIpPrefix = subnetIpParts[1];
1608             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1609             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1610                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1611                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1612             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1613
1614
1615             // Check if external IP is already assigned a route. (i.e. External IP is previously
1616             // allocated to any of the subnets)
1617             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1618             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1619             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1620             if (label != null) {
1621                 //update
1622                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1623                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1624                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1625                     label, internalIp, leastLoadedExtIpAddrStr);
1626                 IpMap newIpm = new IpMapBuilder().setKey(ipMapKey).setInternalIp(internalIp)
1627                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1628                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1629                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1630                 return;
1631             }
1632
1633             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1634             // for the first time and hence not having a route.
1635             //Get the VPN Name using the network ID
1636             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1637             if (vpnName != null) {
1638                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1639                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1640                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1641                         + "added after gateway-set");
1642                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1643                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1644                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1645                         return;
1646                     }
1647                 }
1648                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1649                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1650                     writeFlowInvTx);
1651             }
1652         }
1653     }
1654
1655     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1656         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1657         for (IpMap ipMap : ipMaps) {
1658             if (ipMap.getExternalIp().equals(externalIp)) {
1659                 if (ipMap.getLabel() != null) {
1660                     return ipMap.getLabel();
1661                 }
1662             }
1663         }
1664         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1665         return null;
1666     }
1667
1668     @Override
1669     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1670         LOG.trace("remove : Router delete method");
1671         {
1672             /*
1673             ROUTER DELETE SCENARIO
1674             1) Get the router ID from the event.
1675             2) Build the cookie information from the router ID.
1676             3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1677             4) Build the flow with the cookie value.
1678             5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1679             6) Remove the flows from the other switches which points to the primary and secondary
1680              switches for the flows related the router ID.
1681             7) Get the list of external IP address maintained for the router ID.
1682             8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1683             9) Withdraw the corresponding routes from the BGP.
1684              */
1685
1686             if (identifier == null || router == null) {
1687                 LOG.error("remove : returning without processing since routers is null");
1688                 return;
1689             }
1690
1691             String routerName = router.getRouterName();
1692             if (natMode == NatMode.Conntrack) {
1693                 if (router.isEnableSnat()) {
1694                     centralizedSwitchScheduler.releaseCentralizedSwitch(router);
1695                 }
1696             } else {
1697                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.getKey(),
1698                     () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1699                         LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1700                                 routerName);
1701                         List<ListenableFuture<Void>> futures = new ArrayList<>();
1702                         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1703                         if (routerId == NatConstants.INVALID_ID) {
1704                             LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1705                                     routerName);
1706                             return;
1707                         }
1708                         long bgpVpnId = NatConstants.INVALID_ID;
1709                         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1710                         if (bgpVpnUuid != null) {
1711                             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1712                         }
1713                         addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1714                                 tx);
1715                         Uuid networkUuid = router.getNetworkId();
1716
1717                         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1718                         if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1719                             // No NAPT switch for external router, probably because the router is not attached to
1720                             // any
1721                             // internal networks
1722                             LOG.debug(
1723                                     "No NAPT switch for router {}, check if router is attached to any internal "
1724                                             + "network",
1725                                     routerName);
1726                             return;
1727                         } else {
1728                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1729                             handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1730                                     routerId, tx);
1731                         }
1732                         NatOverVxlanUtil.releaseVNI(routerName, idManager);
1733                     })), NatConstants.NAT_DJC_MAX_RETRIES);
1734             }
1735         }
1736     }
1737
1738     // TODO Clean up the exception handling
1739     @SuppressWarnings("checkstyle:IllegalCatch")
1740     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1741                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1742                                   long routerId, WriteTransaction removeFlowInvTx) {
1743         LOG.info("handleDisableSnat : Entry");
1744         String routerName = router.getRouterName();
1745         try {
1746             if (routerFlag) {
1747                 removeNaptSwitch(routerName);
1748             } else {
1749                 updateNaptSwitch(routerName, BigInteger.ZERO);
1750             }
1751
1752             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1753             naptManager.removeExternalCounter(routerId);
1754
1755             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1756             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1757                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1758                     + "router ID {} from RouterNaptSwitch model", routerId);
1759                 return;
1760             }
1761             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1762                     networkUuid);
1763             if (extNwProvType == null) {
1764                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1765                 return;
1766             }
1767             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1768             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1769                     externalSubnetList, removeFlowInvTx, extNwProvType);
1770             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1771             try {
1772                 String externalSubnetVpn = null;
1773                 for (Uuid externalSubnetId : externalSubnetList) {
1774                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1775                     // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1776                     if (externalSubnet.isPresent()) {
1777                         externalSubnetVpn =  externalSubnetId.getValue();
1778                         clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1779                                 router.getExtGwMacAddress(), removeFlowInvTx);
1780                     }
1781                 }
1782                 if (externalSubnetVpn == null) {
1783                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1784                             router.getExtGwMacAddress(), removeFlowInvTx);
1785                 }
1786             } catch (Exception ex) {
1787                 LOG.error("handleDisableSnat : Failed to remove fib entries for routerId {} in naptSwitchDpnId {}",
1788                     routerId, naptSwitchDpnId, ex);
1789             }
1790             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1791             // for the router ID.
1792             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1793                     + "router ID {} in the DS", routerId);
1794             naptManager.removeMapping(routerId);
1795         } catch (Exception ex) {
1796             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, ex);
1797         }
1798         LOG.info("handleDisableSnat : Exit");
1799     }
1800
1801     // TODO Clean up the exception handling
1802     @SuppressWarnings("checkstyle:IllegalCatch")
1803     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1804                                              @Nonnull Collection<String> externalIps,
1805                                              String vpnId, WriteTransaction writeFlowInvTx) {
1806         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1807                 + "with internet vpn {}", routerName, vpnId);
1808         try {
1809             BigInteger naptSwitchDpnId = null;
1810             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1811                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1812             Optional<RouterToNaptSwitch> rtrToNapt =
1813                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1814             if (rtrToNapt.isPresent()) {
1815                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1816             }
1817             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1818
1819             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1820                     writeFlowInvTx);
1821             try {
1822                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1823                 if (extGwMacAddress != null) {
1824                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1825                             + "External Router ID {}", extGwMacAddress, routerId);
1826                 } else {
1827                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1828                             + "External Router ID {}", routerId);
1829                     return;
1830                 }
1831                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1832                         writeFlowInvTx);
1833             } catch (Exception ex) {
1834                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1835                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1836             }
1837             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1838         } catch (Exception ex) {
1839             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1840                     + "with internet vpn {}", routerName, vpnId, ex);
1841         }
1842         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1843                 routerName, vpnId);
1844     }
1845
1846     // TODO Clean up the exception handling
1847     @SuppressWarnings("checkstyle:IllegalCatch")
1848     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1849         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().setKey(new RouterToNaptSwitchKey(routerName))
1850             .setPrimarySwitchId(naptSwitchId).build();
1851         try {
1852             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1853                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1854         } catch (Exception ex) {
1855             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1856                 naptSwitchId, routerName);
1857         }
1858         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1859             naptSwitchId, routerName);
1860     }
1861
1862     protected void removeNaptSwitch(String routerName) {
1863         // Remove router and switch from model
1864         InstanceIdentifier<RouterToNaptSwitch> id =
1865             InstanceIdentifier.builder(NaptSwitches.class)
1866                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1867         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1868         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1869         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1870         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1871     }
1872
1873     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1874                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1875                                                 @Nonnull Collection<String> externalIps,
1876                                                 Collection<Uuid> externalSubnetList,
1877                                                 WriteTransaction removeFlowInvTx, ProviderTypes extNwProvType) {
1878         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1879         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1880
1881         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1882         // traffic which comes from the  VMs of the NAPT switches)
1883         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1884         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1885
1886         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1887                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1888         mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
1889
1890         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1891         // traffic which comes from the VMs of the non NAPT switches)
1892         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1893                 routerName);
1894         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1895         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1896         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1897                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1898         mdsalManager.removeFlowToTx(tsNatFlowEntity, removeFlowInvTx);
1899
1900         //Remove the flow table 25->44 from NAPT Switch
1901         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1902             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
1903         }
1904
1905         //Remove the Outbound flow entry which forwards the packet to FIB Table
1906         String outboundNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
1907         FlowEntity outboundNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1908             outboundNatFlowRef);
1909
1910         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1911                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1912         mdsalManager.removeFlowToTx(outboundNatFlowEntity, removeFlowInvTx);
1913
1914         removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, removeFlowInvTx);
1915         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1916         // External Subnet Vpn Id.
1917         for (Uuid externalSubnetId : externalSubnetList) {
1918             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1919             if (subnetVpnId != -1) {
1920                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1921                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1922                         natPfibSubnetFlowRef);
1923                 mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1924                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1925                         + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1926                         subnetVpnId, dpnId);
1927             }
1928         }
1929
1930         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1931         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1932         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1933
1934         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1935                 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1936         mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1937
1938         // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1939         // - This does not work since ext-routers is deleted already - no network info
1940         //Get the VPN ID from the ExternalNetworks model
1941         long vpnId = -1;
1942         if (vpnName == null || vpnName.isEmpty()) {
1943             // ie called from router delete cases
1944             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1945             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1946             if (vpnUuid != null) {
1947                 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1948                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete or "
1949                         + "disableSNAT scenario", vpnId, networkId);
1950             }
1951         } else {
1952             // ie called from disassociate vpn case
1953             LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
1954                     vpnName);
1955             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
1956             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}", vpnId);
1957         }
1958
1959         if (vpnId != NatConstants.INVALID_ID) {
1960             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
1961             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
1962             FlowEntity natPfibVpnFlowEntity =
1963                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
1964             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the DPN ID {} "
1965                     + "and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
1966             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, removeFlowInvTx);
1967         }
1968
1969         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
1970         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
1971         if (ipPortMapping == null) {
1972             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
1973             return;
1974         }
1975
1976         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
1977         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1978             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1979             for (IpPortMap ipPortMap : ipPortMaps) {
1980                 String ipPortInternal = ipPortMap.getIpPortInternal();
1981                 String[] ipPortParts = ipPortInternal.split(":");
1982                 if (ipPortParts.length != 2) {
1983                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
1984                     return;
1985                 }
1986                 String internalIp = ipPortParts[0];
1987                 String internalPort = ipPortParts[1];
1988
1989                 //Build the flow for the outbound NAPT table
1990                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
1991                         + NatConstants.COLON_SEPARATOR + internalPort);
1992                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1993                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
1994                 FlowEntity outboundNaptFlowEntity =
1995                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
1996
1997                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
1998                         + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1999                 mdsalManager.removeFlowToTx(outboundNaptFlowEntity, removeFlowInvTx);
2000
2001                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2002                 String externalIp = ipPortExternal.getIpAddress();
2003                 int externalPort = ipPortExternal.getPortNum();
2004
2005                 //Build the flow for the inbound NAPT table
2006                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2007                     String.valueOf(routerId), externalIp, externalPort);
2008                 FlowEntity inboundNaptFlowEntity =
2009                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2010
2011                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2012                         + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2013                 mdsalManager.removeFlowToTx(inboundNaptFlowEntity, removeFlowInvTx);
2014             }
2015         }
2016     }
2017
2018     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2019                                                     @Nonnull Collection<String> externalIps,
2020                                                     WriteTransaction writeFlowInvTx) {
2021         long extVpnId = NatConstants.INVALID_ID;
2022         if (networkId != null) {
2023             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2024             if (vpnUuid != null) {
2025                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2026             } else {
2027                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2028             }
2029         } else {
2030             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2031             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2032         }
2033         if (extVpnId == NatConstants.INVALID_ID) {
2034             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2035             extVpnId = routerId;
2036         }
2037         for (String ip : externalIps) {
2038             String extIp = removeMaskFromIp(ip);
2039             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2040             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2041                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2042                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2043             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2044             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2045         }
2046     }
2047
2048     private String removeMaskFromIp(String ip) {
2049         if (ip != null && !ip.trim().isEmpty()) {
2050             return ip.split("/")[0];
2051         }
2052         return ip;
2053     }
2054
2055     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2056                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2057                                                            WriteTransaction writeFlowInvTx) {
2058         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2059         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2060
2061         //Remove the NAPT PFIB TABLE entry
2062         long vpnId = -1;
2063         if (vpnName != null) {
2064             // ie called from disassociate vpn case
2065             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2066                     + "with vpnName {}", vpnName);
2067             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2068             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2069                     vpnId);
2070         }
2071
2072         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2073                 networkId, routerName, dpnId)) {
2074             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2075             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2076             FlowEntity natPfibVpnFlowEntity =
2077                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2078             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2079                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2080             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2081
2082             // Remove IP-PORT active NAPT entries and release port from IdManager
2083             // For the router ID get the internal IP , internal port and the corresponding
2084             // external IP and external Port.
2085             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2086             if (ipPortMapping == null) {
2087                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2088                 return;
2089             }
2090             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2091             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2092                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2093                 for (IpPortMap ipPortMap : ipPortMaps) {
2094                     String ipPortInternal = ipPortMap.getIpPortInternal();
2095                     String[] ipPortParts = ipPortInternal.split(":");
2096                     if (ipPortParts.length != 2) {
2097                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2098                                 + "and port");
2099                         return;
2100                     }
2101                     String internalIp = ipPortParts[0];
2102                     String internalPort = ipPortParts[1];
2103
2104                     //Build the flow for the outbound NAPT table
2105                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2106                             + NatConstants.COLON_SEPARATOR + internalPort);
2107                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2108                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2109                     FlowEntity outboundNaptFlowEntity =
2110                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2111
2112                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2113                             + "active switch with the DPN ID {} and router ID {}",
2114                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2115                     mdsalManager.removeFlowToTx(outboundNaptFlowEntity, writeFlowInvTx);
2116
2117                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2118                     String externalIp = ipPortExternal.getIpAddress();
2119                     int externalPort = ipPortExternal.getPortNum();
2120
2121                     //Build the flow for the inbound NAPT table
2122                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2123                         String.valueOf(routerId), externalIp, externalPort);
2124                     FlowEntity inboundNaptFlowEntity =
2125                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2126
2127                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2128                             + "active active switch with the DPN ID {} and router ID {}",
2129                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2130                     mdsalManager.removeFlowToTx(inboundNaptFlowEntity, writeFlowInvTx);
2131
2132                     // Finally release port from idmanager
2133                     String internalIpPort = internalIp + ":" + internalPort;
2134                     naptManager.removePortFromPool(internalIpPort, externalIp);
2135
2136                     //Remove sessions from models
2137                     naptManager.removeIpPortMappingForRouterID(routerId);
2138                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2139                 }
2140             }
2141         } else {
2142             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2143         }
2144     }
2145
2146     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2147             BigInteger naptSwitchDpnId, WriteTransaction removeFlowInvTx) {
2148         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2149
2150         // Remove the flows from the other switches which points to the primary and secondary switches
2151         // for the flows related the router ID.
2152         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2153         if (allSwitchList.isEmpty()) {
2154             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2155             return;
2156         }
2157         for (BigInteger dpnId : allSwitchList) {
2158             if (!naptSwitchDpnId.equals(dpnId)) {
2159                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2160
2161                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2162                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2163                 FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2164
2165                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2166                         + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2167                 mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
2168
2169                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2170                 long groupId = createGroupId(getGroupIdKey(routerName));
2171                 List<BucketInfo> listBucketInfo = new ArrayList<>();
2172                 GroupEntity preSnatGroupEntity =
2173                     MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, listBucketInfo);
2174
2175                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2176                         + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2177                 mdsalManager.removeGroup(preSnatGroupEntity);
2178
2179             }
2180         }
2181     }
2182
2183     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2184                                          @Nonnull Collection<String> externalIps, String vpnName,
2185                                          String extGwMacAddress, WriteTransaction removeFlowInvTx) {
2186         //Withdraw the corresponding routes from the BGP.
2187         //Get the network ID using the router ID.
2188         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2189                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2190         if (networkUuid == null) {
2191             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2192             return;
2193         }
2194
2195         if (externalIps.isEmpty()) {
2196             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2197             return;
2198         }
2199
2200         if (vpnName == null) {
2201             //Get the VPN Name using the network ID
2202             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2203             if (vpnName == null) {
2204                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2205                     networkUuid, routerId);
2206                 return;
2207             }
2208         }
2209         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2210
2211         //Remove custom FIB routes
2212         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2213         for (String extIp : externalIps) {
2214             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, removeFlowInvTx);
2215         }
2216     }
2217
2218     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2219                                             final Uuid networkUuid, String extGwMacAddress,
2220                                             WriteTransaction removeFlowInvTx) {
2221         clearBgpRoutes(extIp, vpnName);
2222         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2223                 removeFlowInvTx);
2224     }
2225
2226     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2227                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2228                                              String gwMacAddress, boolean switchOver,
2229                                              WriteTransaction removeFlowInvTx) {
2230         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2231         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2232         if (routerName == null) {
2233             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2234             return;
2235         }
2236         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2237         if (extNwProvType == null) {
2238             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2239             return;
2240         }
2241         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2242          * external network provided type is VxLAN
2243          */
2244         if (extNwProvType == ProviderTypes.VXLAN) {
2245             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress,
2246                     removeFlowInvTx);
2247             return;
2248         }
2249         if (tempLabel < 0) {
2250             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2251                     extIp, routerId);
2252             return;
2253         }
2254
2255         final long label = tempLabel;
2256         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2257         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2258                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2259                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2260         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2261
2262         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2263         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2264         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2265             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2266             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2267         }
2268         if (!switchOver) {
2269             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2270                     Futures.transformAsync(future, result -> {
2271                                 //Release label
2272                         if (result.isSuccessful()) {
2273                             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2274                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2275                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2276                             return vpnService.removeVpnLabel(labelInput);
2277                         } else {
2278                             String errMsg =
2279                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2280                                             + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2281                             LOG.error(errMsg);
2282                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2283                         }
2284                     }, MoreExecutors.directExecutor());
2285
2286             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2287
2288                 @Override
2289                 public void onFailure(@Nonnull Throwable error) {
2290                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2291                         + "got external ip {}", label, extIp, error);
2292                 }
2293
2294                 @Override
2295                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2296                     if (result.isSuccessful()) {
2297                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2298                             + "from VPN {}", externalIp, vpnName);
2299                     } else {
2300                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2301                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2302                     }
2303                 }
2304             }, MoreExecutors.directExecutor());
2305         } else {
2306             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2307                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2308         }
2309     }
2310
2311     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2312                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2313                                            WriteTransaction removeFlowInvTx) {
2314         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2315         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2316         if (routerName == null) {
2317             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2318             return;
2319         }
2320         //Get the external network provider type from networkId
2321         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2322         if (extNwProvType == null) {
2323             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2324                     networkUuid);
2325             return;
2326         }
2327         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2328          *  external network provided type is VxLAN
2329          */
2330         if (extNwProvType == ProviderTypes.VXLAN) {
2331             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress,
2332                     removeFlowInvTx);
2333             return;
2334         }
2335         //Get IPMaps from the DB for the router ID
2336         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2337         if (dbIpMaps.isEmpty()) {
2338             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2339             return;
2340         }
2341
2342         long tempLabel = NatConstants.INVALID_ID;
2343         for (IpMap dbIpMap : dbIpMaps) {
2344             String dbExternalIp = dbIpMap.getExternalIp();
2345             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2346             //Select the IPMap, whose external IP is the IP for which FIB is installed
2347             if (extIp.equals(dbExternalIp)) {
2348                 tempLabel = dbIpMap.getLabel();
2349                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2350                     tempLabel, dbExternalIp, routerId);
2351                 break;
2352             }
2353         }
2354         if (tempLabel == NatConstants.INVALID_ID) {
2355             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2356                     extIp, routerId);
2357             return;
2358         }
2359
2360         final long label = tempLabel;
2361         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2362         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2363                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2364                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2365         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2366
2367         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2368         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2369         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2370             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2371             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2372         }
2373         if (!switchOver) {
2374             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2375                     Futures.transformAsync(future, result -> {
2376                         //Release label
2377                         if (result.isSuccessful()) {
2378                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2379                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2380                             return vpnService.removeVpnLabel(labelInput);
2381                         } else {
2382                             String errMsg =
2383                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2384                                             + "prefix %s Failed - %s",
2385                                             dpnId, externalIp, result.getErrors());
2386                             LOG.error(errMsg);
2387                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2388                         }
2389                     }, MoreExecutors.directExecutor());
2390
2391             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2392
2393                 @Override
2394                 public void onFailure(@Nonnull Throwable error) {
2395                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2396                 }
2397
2398                 @Override
2399                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2400                     if (result.isSuccessful()) {
2401                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2402                             + "from VPN {}", externalIp, vpnName);
2403                     } else {
2404                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2405                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2406                     }
2407                 }
2408             }, MoreExecutors.directExecutor());
2409         } else {
2410             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2411                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2412         }
2413     }
2414
2415     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2416                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2417                                                WriteTransaction writeFlowInvTx) {
2418         //Withdraw the corresponding routes from the BGP.
2419         //Get the network ID using the router ID.
2420         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2421                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2422         if (networkUuid == null) {
2423             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2424             return;
2425         }
2426
2427         if (externalIps == null || externalIps.isEmpty()) {
2428             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2429             return;
2430         }
2431
2432         if (vpnName == null) {
2433             //Get the VPN Name using the network ID
2434             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2435             if (vpnName == null) {
2436                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2437                     networkUuid, routerId);
2438                 return;
2439             }
2440         }
2441         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2442
2443         //Remove custom FIB routes
2444         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2445         for (String extIp : externalIps) {
2446             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2447                     writeFlowInvTx);
2448         }
2449     }
2450
2451     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2452         //Inform BGP about the route removal
2453         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2454         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2455         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2456     }
2457
2458     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2459         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2460         List<MatchInfo> mkMatches = new ArrayList<>();
2461         // Matching metadata
2462         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
2463         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
2464             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
2465             5, String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
2466             COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
2467         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2468         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2469     }
2470
2471     private void removeLFibTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2472         List<MatchInfo> matches = new ArrayList<>();
2473         matches.add(MatchEthernetType.MPLS_UNICAST);
2474         matches.add(new MatchMplsLabel(serviceId));
2475
2476         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2477
2478         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2479
2480         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
2481             10, flowRef, 0, 0,
2482             COOKIE_VM_LFIB_TABLE, matches, null);
2483
2484         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2485
2486         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2487     }
2488
2489     /**
2490      * router association to vpn.
2491      *
2492      * @param routerName - Name of router
2493      * @param routerId - router id
2494      * @param bgpVpnName BGP VPN name
2495      */
2496     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2497             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2498         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2499         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2500             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2501
2502             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2503
2504             if (bgpVpnId != NatConstants.INVALID_ID) {
2505                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2506                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2507                 RouterIds rtrs = new RouterIdsBuilder().setKey(new RouterIdsKey(bgpVpnId))
2508                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2509                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2510                     getRoutersIdentifier(bgpVpnId), rtrs);
2511
2512                 // Get the allocated Primary NAPT Switch for this router
2513                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2514
2515                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2516                         routerId, bgpVpnId);
2517                 addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, true, writeFlowInvTx);
2518
2519                 // Get the group ID
2520                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2521                 createGroupId(getGroupIdKey(routerName));
2522                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2523                         extNwProvType);
2524             }
2525         }
2526     }
2527
2528     /**
2529      * router disassociation from vpn.
2530      *
2531      * @param routerName - Name of router
2532      * @param routerId - router id
2533      * @param bgpVpnName BGP VPN name
2534      */
2535     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2536             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2537         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2538         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2539             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2540             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2541
2542             // Get the allocated Primary NAPT Switch for this router
2543             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2544
2545             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2546             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID,
2547                     true, writeFlowInvTx);
2548
2549             // Get the group ID
2550             createGroupId(getGroupIdKey(routerName));
2551             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2552             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2553                     writeFlowInvTx, extNwProvType);
2554         }
2555     }
2556
2557     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2558         InstanceIdentifier<Routers> routerInstanceIndentifier =
2559             InstanceIdentifier.builder(ExtRouters.class)
2560                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2561         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2562         return routerData.isPresent() && routerData.get().isEnableSnat();
2563     }
2564
2565     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2566                                              long routerId, boolean isSnatCfgd, WriteTransaction writeFlowInvTx,
2567                                              ProviderTypes extNwProvType) {
2568         long changedVpnId = bgpVpnId;
2569         String idType = "BGP VPN";
2570         if (bgpVpnId == NatConstants.INVALID_ID) {
2571             changedVpnId = routerId;
2572             idType = "router";
2573         }
2574
2575         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2576         if (switches.isEmpty()) {
2577             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2578             return;
2579         }
2580         for (BigInteger dpnId : switches) {
2581             // Update the BGP VPN ID in the SNAT miss entry to group
2582             if (!dpnId.equals(primarySwitchId)) {
2583                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2584                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2585                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2586                 long groupId = createGroupId(getGroupIdKey(routerName));
2587                 if (!isSnatCfgd) {
2588                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2589                 }
2590
2591                 LOG.debug(
2592                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2593                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2594                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2595                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2596             } else {
2597                 LOG.debug(
2598                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2599                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2600                 FlowEntity flowEntity =
2601                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2602                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2603
2604                 LOG.debug(
2605                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2606                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2607                         idType, changedVpnId, primarySwitchId);
2608                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2609                         changedVpnId, writeFlowInvTx, extNwProvType);
2610
2611                 LOG.debug(
2612                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2613                                 + "which punts the packet to the controller in the Primary switch {}",
2614                         idType, changedVpnId, primarySwitchId);
2615                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2616
2617                 LOG.debug(
2618                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2619                                 + " outgoing packet to FIB Table in the Primary switch {}",
2620                         idType, changedVpnId, primarySwitchId);
2621                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2622
2623                 LOG.debug(
2624                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2625                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2626                                 + " {}", idType, changedVpnId, primarySwitchId);
2627                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2628
2629                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2630                         primarySwitchId);
2631                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2632                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2633                 if (vpnId != NatConstants.INVALID_ID) {
2634                     installNaptPfibEntry(primarySwitchId, vpnId, writeFlowInvTx);
2635                 }
2636             }
2637         }
2638     }
2639
2640     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2641         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2642         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2643         if (ipPortMapping == null) {
2644             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2645             return;
2646         }
2647         // Get the External Gateway MAC Address
2648         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2649         if (extGwMacAddress != null) {
2650             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2651                     extGwMacAddress, routerId);
2652         } else {
2653             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2654                     routerId);
2655             return;
2656         }
2657         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2658         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2659             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2660             for (IpPortMap ipPortMap : ipPortMaps) {
2661                 String ipPortInternal = ipPortMap.getIpPortInternal();
2662                 String[] ipPortParts = ipPortInternal.split(":");
2663                 if (ipPortParts.length != 2) {
2664                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2665                     return;
2666                 }
2667                 String internalIp = ipPortParts[0];
2668                 String internalPort = ipPortParts[1];
2669                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2670                         internalIp, internalPort);
2671                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2672                 NAPTEntryEvent.Protocol protocol;
2673                 switch (protocolTypes) {
2674                     case TCP:
2675                         protocol = NAPTEntryEvent.Protocol.TCP;
2676                         break;
2677                     case UDP:
2678                         protocol = NAPTEntryEvent.Protocol.UDP;
2679                         break;
2680                     default:
2681                         protocol = NAPTEntryEvent.Protocol.TCP;
2682                 }
2683                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2684                 SessionAddress externalAddress =
2685                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2686                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2687                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2688                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2689                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2690                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2691             }
2692         }
2693     }
2694
2695     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2696                                                           long changedVpnId) {
2697
2698         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2699             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2700         List<MatchInfo> matches = new ArrayList<>();
2701         matches.add(MatchEthernetType.IPV4);
2702         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2703
2704         List<ActionInfo> actionsInfo = new ArrayList<>();
2705         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2706                 routerName);
2707         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2708         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2709                 actionsInfo);
2710         actionsInfo.add(new ActionGroup(groupId));
2711         List<InstructionInfo> instructions = new ArrayList<>();
2712         instructions.add(new InstructionApplyActions(actionsInfo));
2713         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2714         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2715             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2716             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2717
2718         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2719         return flowEntity;
2720     }
2721
2722     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2723                                                                         long changedVpnId) {
2724
2725         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2726                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2727         List<MatchInfo> matches = new ArrayList<>();
2728         matches.add(MatchEthernetType.IPV4);
2729         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2730
2731         List<InstructionInfo> instructions = new ArrayList<>();
2732         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2733
2734         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2735         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2736             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2737             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2738
2739         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2740         return flowEntity;
2741     }
2742
2743     // TODO : Replace this with ITM Rpc once its available with full functionality
2744     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2745             long routerId, long changedVpnId, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2746         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2747             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2748         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2749                 extNwProvType);
2750         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2751     }
2752
2753     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2754             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2755         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2756             dpId, routerName, changedVpnId);
2757         List<MatchInfo> matches = new ArrayList<>();
2758         matches.add(MatchEthernetType.IPV4);
2759
2760         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2761         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2762             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2763         }
2764         matches.add(new MatchTunnelId(tunnelId));
2765
2766         List<InstructionInfo> instructions = new ArrayList<>();
2767         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2768             MetaDataUtil.METADATA_MASK_VRFID));
2769         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2770         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2771         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2772         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2773             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2774             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2775         return flowEntity;
2776     }
2777
2778     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2779                                                  WriteTransaction writeFlowInvTx) {
2780         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2781             dpnId, routerId, changedVpnId);
2782         FlowEntity flowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId);
2783         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing flow {}", flowEntity);
2784         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2785     }
2786
2787     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId, long changedVpnId) {
2788         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2789             dpId, routerId, changedVpnId);
2790         List<MatchInfo> matches = new ArrayList<>();
2791         matches.add(MatchEthernetType.IPV4);
2792         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2793
2794         List<InstructionInfo> instructions = new ArrayList<>();
2795         List<ActionInfo> actionsInfos = new ArrayList<>();
2796         actionsInfos.add(new ActionPuntToController());
2797         instructions.add(new InstructionApplyActions(actionsInfos));
2798         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2799             MetaDataUtil.METADATA_MASK_VRFID));
2800
2801         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
2802         BigInteger cookie = getCookieOutboundFlow(routerId);
2803         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2804             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2805         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2806         return flowEntity;
2807     }
2808
2809     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2810                                                WriteTransaction writeFlowInvTx) {
2811         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2812             dpnId, segmentId, changedVpnId);
2813         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2814         mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
2815     }
2816
2817     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2818
2819         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2820             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2821         List<MatchInfo> matches = new ArrayList<>();
2822         matches.add(MatchEthernetType.IPV4);
2823         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2824
2825         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2826         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2827         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2828         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2829         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2830
2831         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2832         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2833             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2834             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2835         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2836         return flowEntity;
2837     }
2838
2839     @Override
2840     protected ExternalRoutersListener getDataTreeChangeListener() {
2841         return ExternalRoutersListener.this;
2842     }
2843
2844     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2845                                                             WriteTransaction writeFlowInvTx) {
2846         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2847                 routerName);
2848         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2849             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2850             if (subnetVpnId != -1) {
2851                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2852                     + "and vpnId {}", dpnId, subnetVpnId);
2853                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2854             }
2855         }
2856     }
2857 }