natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.AsyncFunction;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import java.math.BigInteger;
  18 import java.net.Inet6Address;
  19 import java.net.InetAddress;
  20 import java.net.UnknownHostException;
  21 import java.util.ArrayList;
  22 import java.util.Collection;
  23 import java.util.Collections;
  24 import java.util.HashMap;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ExecutionException;
  31 import java.util.concurrent.Future;
  32 import javax.annotation.Nonnull;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  38 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  41 import org.opendaylight.genius.mdsalutil.ActionInfo;
  42 import org.opendaylight.genius.mdsalutil.BucketInfo;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.GroupEntity;
  45 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.mdsalutil.MatchInfo;
  48 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.UpgradeState;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  58 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  59 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  62 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  63 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  64 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  65 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  66 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  67 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  68 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  69 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  70 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  71 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  72 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  74 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 136 import org.opendaylight.yangtools.yang.binding.DataObject;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 138 import org.opendaylight.yangtools.yang.common.RpcResult;
 139 import org.slf4j.Logger;
 140 import org.slf4j.LoggerFactory;
 141
 142 @Singleton
 143 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 144     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 145
 146     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 147     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 148
 149     private final DataBroker dataBroker;
 150     private final IMdsalApiManager mdsalManager;
 151     private final ItmRpcService itmManager;
 152     private final OdlInterfaceRpcService interfaceManager;
 153     private final IdManagerService idManager;
 154     private final NaptManager naptManager;
 155     private final NAPTSwitchSelector naptSwitchSelector;
 156     private final IBgpManager bgpManager;
 157     private final VpnRpcService vpnService;
 158     private final FibRpcService fibService;
 159     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 160     private final NaptEventHandler naptEventHandler;
 161     private final NaptPacketInHandler naptPacketInHandler;
 162     private final IFibManager fibManager;
 163     private final IVpnManager vpnManager;
 164     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 165     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 166     private final NatMode natMode;
 167     private final INeutronVpnManager nvpnManager;
 168     private final IElanService elanManager;
 169     private final JobCoordinator coordinator;
 170     private final UpgradeState upgradeState;
 171
 172     @Inject
 173     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 174                                    final ItmRpcService itmManager,
 175                                    final OdlInterfaceRpcService interfaceManager,
 176                                    final IdManagerService idManager,
 177                                    final NaptManager naptManager,
 178                                    final NAPTSwitchSelector naptSwitchSelector,
 179                                    final IBgpManager bgpManager,
 180                                    final VpnRpcService vpnService,
 181                                    final FibRpcService fibService,
 182                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 183                                    final NaptEventHandler naptEventHandler,
 184                                    final NaptPacketInHandler naptPacketInHandler,
 185                                    final IFibManager fibManager,
 186                                    final IVpnManager vpnManager,
 187                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 188                                    final INeutronVpnManager nvpnManager,
 189                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 190                                    final NatserviceConfig config,
 191                                    final IElanService elanManager,
 192                                    final JobCoordinator coordinator,
 193                                    final UpgradeState upgradeState) {
 194         super(Routers.class, ExternalRoutersListener.class);
 195         this.dataBroker = dataBroker;
 196         this.mdsalManager = mdsalManager;
 197         this.itmManager = itmManager;
 198         this.interfaceManager = interfaceManager;
 199         this.idManager = idManager;
 200         this.naptManager = naptManager;
 201         this.naptSwitchSelector = naptSwitchSelector;
 202         this.bgpManager = bgpManager;
 203         this.vpnService = vpnService;
 204         this.fibService = fibService;
 205         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 206         this.naptEventHandler = naptEventHandler;
 207         this.naptPacketInHandler = naptPacketInHandler;
 208         this.fibManager = fibManager;
 209         this.vpnManager = vpnManager;
 210         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 211         this.nvpnManager = nvpnManager;
 212         this.elanManager = elanManager;
 213         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 214         this.coordinator = coordinator;
 215         this.upgradeState = upgradeState;
 216         if (config != null) {
 217             this.natMode = config.getNatMode();
 218         } else {
 219             this.natMode = NatMode.Controller;
 220         }
 221     }
 222
 223     @Override
 224     @PostConstruct
 225     public void init() {
 226         LOG.info("{} init", getClass().getSimpleName());
 227         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 228         createGroupIdPool();
 229     }
 230
 231     @Override
 232     protected InstanceIdentifier<Routers> getWildCardPath() {
 233         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 234     }
 235
 236     @Override
 237     // TODO Clean up the exception handling
 238     @SuppressWarnings("checkstyle:IllegalCatch")
 239     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 240         // Populate the router-id-name container
 241         String routerName = routers.getRouterName();
 242         LOG.info("add : external router event for {}", routerName);
 243         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 244         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 245         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 246         if (natMode == NatMode.Conntrack && !upgradeState.isUpgradeInProgress()) {
 247             if (bgpVpnUuid != null) {
 248                 return;
 249             }
 250             List<ExternalIps> externalIps = routers.getExternalIps();
 251             // Allocate Primary Napt Switch for this router
 252             if (routers.isEnableSnat() && externalIps != null && !externalIps.isEmpty()) {
 253                 centralizedSwitchScheduler.scheduleCentralizedSwitch(routers);
 254             }
 255             //snatServiceManger.notify(routers, null, Action.ADD);
 256         } else {
 257             try {
 258                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.getKey(), () -> {
 259                     WriteTransaction writeFlowInvTx = dataBroker.newWriteOnlyTransaction();
 260                     LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 261                     long bgpVpnId = NatConstants.INVALID_ID;
 262                     if (bgpVpnUuid != null) {
 263                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 264                     }
 265                     addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, writeFlowInvTx);
 266                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 267                     // Allocate Primary Napt Switch for this router
 268                     BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 269                     if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 270                         if (!routers.isEnableSnat()) {
 271                             LOG.info("add : SNAT is disabled for external router {} ", routerName);
 272                             /* If SNAT is disabled on ext-router though L3_FIB_TABLE(21) -> PSNAT_TABLE(26) flow
 273                              * is required for DNAT. Hence writeFlowInvTx object submit is required.
 274                              */
 275                             return futures;
 276                         }
 277                         handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, writeFlowInvTx);
 278                     }
 279                     //final submit call for writeFlowInvTx
 280                     futures.add(NatUtil.waitForTransactionToComplete(writeFlowInvTx));
 281                     return futures;
 282                 }, NatConstants.NAT_DJC_MAX_RETRIES);
 283             } catch (Exception ex) {
 284                 LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 285                         routerName, ex);
 286             }
 287         }
 288     }
 289
 290     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 291                                  WriteTransaction writeFlowInvTx) {
 292         String routerName = routers.getRouterName();
 293         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 294
 295         naptManager.initialiseExternalCounter(routers, routerId);
 296         subnetRegisterMapping(routers, routerId);
 297
 298         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 299             primarySwitchId, routerName);
 300
 301         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 302                 routers.getNetworkId());
 303         if (extNwProvType == null) {
 304             LOG.error("handleEnableSnat : External Network Provider Type missing");
 305             return;
 306         }
 307
 308         if (bgpVpnId != NatConstants.INVALID_ID) {
 309             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, writeFlowInvTx,
 310                     extNwProvType);
 311         } else {
 312             // write metadata and punt
 313             installOutboundMissEntry(routerName, routerId, primarySwitchId, writeFlowInvTx);
 314             // Now install entries in SNAT tables to point to Primary for each router
 315             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 316             for (BigInteger dpnId : switches) {
 317                 // Handle switches and NAPT switches separately
 318                 if (!dpnId.equals(primarySwitchId)) {
 319                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 320                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 321                 } else {
 322                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 323                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, writeFlowInvTx);
 324                 }
 325             }
 326         }
 327
 328         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 329         if (externalIps.isEmpty()) {
 330             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 331             return;
 332         } else {
 333             for (String externalIpAddrPrefix : externalIps) {
 334                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 335                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 336                 handleSnatReverseTraffic(primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix,
 337                         writeFlowInvTx);
 338             }
 339         }
 340         LOG.debug("handleEnableSnat : Exit");
 341     }
 342
 343     private BigInteger getPrimaryNaptSwitch(String routerName) {
 344         // Allocate Primary Napt Switch for this router
 345         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 346         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 347             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 348                 primarySwitchId, routerName);
 349             return primarySwitchId;
 350         }
 351         // Validating and creating VNI pool during when NAPT switch is selected.
 352         // With Assumption this might be the first NAT service comes up.
 353         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 354                 NatConstants.ODL_VNI_POOL_NAME);
 355         // Allocated an id from VNI pool for the Router.
 356         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 357         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 358         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 359
 360         return primarySwitchId;
 361     }
 362
 363     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 364                                                      WriteTransaction writeFlowInvTx) {
 365         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 366         if (extVpnId == NatConstants.INVALID_ID) {
 367             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 368             extVpnId = routerId;
 369         }
 370         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 371         if (externalIps.isEmpty()) {
 372             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 373                 dpnId, extVpnId);
 374             return;
 375         }
 376         for (String ip : externalIps) {
 377             Uuid subnetId = getSubnetIdForFixedIp(ip);
 378             if (subnetId != null) {
 379                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 380                 if (subnetVpnId != NatConstants.INVALID_ID) {
 381                     extVpnId = subnetVpnId;
 382                 }
 383                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 384                     dpnId, extVpnId, subnetId);
 385                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 386                 mdsalManager.addFlowToTx(postNaptFlowEntity, writeFlowInvTx);
 387             }
 388         }
 389     }
 390
 391     private Uuid getSubnetIdForFixedIp(String ip) {
 392         if (ip != null) {
 393             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 394             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 395             Uuid subnetId = NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 396             return subnetId;
 397         }
 398         LOG.error("getSubnetIdForFixedIp : ip is null");
 399         return null;
 400     }
 401
 402     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 403         List<Uuid> subnetList = null;
 404         List<String> externalIps = null;
 405         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 406         subnetList = routerEntry.getSubnetIds();
 407         externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 408         int counter = 0;
 409         int extIpCounter = externalIps.size();
 410         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 411                 counter, extIpCounter);
 412         for (Uuid subnet : subnetList) {
 413             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 414             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 415                 .builder(Subnetmaps.class)
 416                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 417                 .build();
 418             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 419             if (sn.isPresent()) {
 420                 // subnets
 421                 Subnetmap subnetmapEntry = sn.get();
 422                 String subnetString = subnetmapEntry.getSubnetIp();
 423                 String[] subnetSplit = subnetString.split("/");
 424                 String subnetIp = subnetSplit[0];
 425                 try {
 426                     InetAddress address = InetAddress.getByName(subnetIp);
 427                     if (address instanceof Inet6Address) {
 428                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 429                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 430                         continue;
 431                     }
 432                 } catch (UnknownHostException e) {
 433                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 434                     return;
 435                 }
 436                 String subnetPrefix = "0";
 437                 if (subnetSplit.length == 2) {
 438                     subnetPrefix = subnetSplit[1];
 439                 }
 440                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 441                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 442                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 443                 //externalIps
 444                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 445                         counter, extIpCounter);
 446                 if (extIpCounter != 0) {
 447                     if (counter < extIpCounter) {
 448                         String[] ipSplit = externalIps.get(counter).split("/");
 449                         String externalIp = ipSplit[0];
 450                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 451                         if (ipSplit.length == 2) {
 452                             extPrefix = ipSplit[1];
 453                         }
 454                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 455                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 456                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 457                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 458                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 459                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 460                     } else {
 461                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 462                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 463                         String[] ipSplit = externalIps.get(counter).split("/");
 464                         String externalIp = ipSplit[0];
 465                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 466                         if (ipSplit.length == 2) {
 467                             extPrefix = ipSplit[1];
 468                         }
 469                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 470                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 471                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 472                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 473                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 474                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 475                             externalIp, extPrefix);
 476                     }
 477                 }
 478                 counter++;
 479                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 480             } else {
 481                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 482             }
 483         }
 484     }
 485
 486     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 487             Uuid bgpVpnUuid, boolean create, WriteTransaction writeFlowInvTx) {
 488         //Check if BGP VPN exists. If exists then invoke the new method.
 489         if (bgpVpnId != NatConstants.INVALID_ID) {
 490             if (bgpVpnUuid != null) {
 491                 String bgpVpnName = bgpVpnUuid.getValue();
 492                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 493                     bgpVpnId, bgpVpnName);
 494                 RouterIds rtrs = new RouterIdsBuilder().setKey(new RouterIdsKey(bgpVpnId))
 495                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 496                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 497                     getRoutersIdentifier(bgpVpnId), rtrs);
 498             }
 499             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, create, writeFlowInvTx);
 500             return;
 501         }
 502
 503         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 504         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, writeFlowInvTx);
 505     }
 506
 507     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 508             WriteTransaction writeFlowInvTx) {
 509         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 510         if (switches.isEmpty()) {
 511             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 512             return;
 513         }
 514         if (routerId == NatConstants.INVALID_ID) {
 515             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 516                     + "default NAT route in FIB", routerName);
 517             return;
 518         }
 519         for (BigInteger dpnId : switches) {
 520             if (create) {
 521                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 522                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 523                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 524             } else {
 525                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 526                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 527                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 528             }
 529         }
 530     }
 531
 532     private void addOrDelDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 533             long bgpVpnId, boolean create,WriteTransaction writeFlowInvTx) {
 534         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 535         if (dpnIds.isEmpty()) {
 536             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 537                     + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 538             return;
 539         }
 540         for (BigInteger dpnId : dpnIds) {
 541             if (create) {
 542                 if (bgpVpnId != NatConstants.INVALID_ID) {
 543                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 544                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 545                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 546                 } else {
 547                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 548                             + "in dpn {} for the internal vpn", routerId, dpnId);
 549                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId,writeFlowInvTx);
 550                 }
 551             } else {
 552                 if (bgpVpnId != NatConstants.INVALID_ID) {
 553                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 554                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 555                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 556                 } else {
 557                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 558                             + "in dpn {} for the internal vpn", routerId, dpnId);
 559                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 560                 }
 561             }
 562         }
 563     }
 564
 565     // TODO Clean up the exception handling
 566     @SuppressWarnings("checkstyle:IllegalCatch")
 567     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 568                                                           InstanceIdentifier<T> path) {
 569         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 570
 571         try {
 572             return tx.read(datastoreType, path).get();
 573         } catch (Exception e) {
 574             throw new RuntimeException(e);
 575         }
 576     }
 577
 578     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 579                                             WriteTransaction writeFlowInvTx) {
 580         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 581         if (routerId != NatConstants.INVALID_ID) {
 582             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 583                     primarySwitchId, routerId);
 584             createOutboundTblEntry(primarySwitchId, routerId, writeFlowInvTx);
 585         } else {
 586             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 587                 + "createAndInstallMissEntry", routerName);
 588         }
 589     }
 590
 591     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID) {
 592         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 593                 .FLOWID_SEPARATOR + routerID;
 594     }
 595
 596     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 597         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 598                 .FLOWID_SEPARATOR + vpnId;
 599     }
 600
 601     public BigInteger getCookieOutboundFlow(long routerId) {
 602         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 603             BigInteger.valueOf(routerId));
 604     }
 605
 606     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId) {
 607         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 608         List<MatchInfo> matches = new ArrayList<>();
 609         matches.add(MatchEthernetType.IPV4);
 610         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 611
 612         List<InstructionInfo> instructions = new ArrayList<>();
 613         List<ActionInfo> actionsInfos = new ArrayList<>();
 614         actionsInfos.add(new ActionPuntToController());
 615         instructions.add(new InstructionApplyActions(actionsInfos));
 616         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 617             MetaDataUtil.METADATA_MASK_VRFID));
 618
 619         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
 620         BigInteger cookie = getCookieOutboundFlow(routerId);
 621         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 622             5, flowRef, 0, 0,
 623             cookie, matches, instructions);
 624         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 625         return flowEntity;
 626     }
 627
 628     public void createOutboundTblEntry(BigInteger dpnId, long routerId, WriteTransaction writeFlowInvTx) {
 629         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 630         FlowEntity flowEntity = buildOutboundFlowEntity(dpnId, routerId);
 631         LOG.debug("createOutboundTblEntry : Installing flow {}", flowEntity);
 632         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 633     }
 634
 635     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 636         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 637         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 638         try {
 639             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 640                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 641                     .setSourceDpid(srcDpId)
 642                     .setDestinationDpid(dstDpId)
 643                     .setTunnelType(tunType)
 644                     .build());
 645             rpcResult = result.get();
 646             if (!rpcResult.isSuccessful()) {
 647                 tunType = TunnelTypeGre.class;
 648                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 649                     .setSourceDpid(srcDpId)
 650                     .setDestinationDpid(dstDpId)
 651                     .setTunnelType(tunType)
 652                     .build());
 653                 rpcResult = result.get();
 654                 if (!rpcResult.isSuccessful()) {
 655                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 656                             rpcResult.getErrors());
 657                 } else {
 658                     return rpcResult.getResult().getInterfaceName();
 659                 }
 660                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 661                         rpcResult.getErrors());
 662             } else {
 663                 return rpcResult.getResult().getInterfaceName();
 664             }
 665         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 666             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 667                     + "between {} and {}", srcDpId, dstDpId, e);
 668         }
 669
 670         return null;
 671     }
 672
 673     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 674                                                     WriteTransaction writeFlowInvTx) {
 675         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 676         // Install miss entry pointing to group
 677         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 678         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 679     }
 680
 681     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 682             String routerName, long routerId) {
 683         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 684             dpnId, bucketInfo.get(0));
 685         // Install the select group
 686         long groupId = createGroupId(getGroupIdKey(routerName));
 687         GroupEntity groupEntity =
 688             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 689         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 690         mdsalManager.syncInstallGroup(groupEntity);
 691         // Install miss entry pointing to group
 692         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 693         if (flowEntity == null) {
 694             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 695                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 696                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 697             return;
 698         }
 699         mdsalManager.installFlow(flowEntity);
 700     }
 701
 702     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 703         long groupId = createGroupId(getGroupIdKey(routerName));
 704         GroupEntity groupEntity =
 705             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 706         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 707         mdsalManager.syncInstallGroup(groupEntity);
 708         return groupId;
 709     }
 710
 711     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 712         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 713                 dpId, routerName, groupId);
 714         List<MatchInfo> matches = new ArrayList<>();
 715         matches.add(MatchEthernetType.IPV4);
 716         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 717
 718         List<ActionInfo> actionsInfo = new ArrayList<>();
 719         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 720                 routerName);
 721         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 722         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 723         actionsInfo.add(new ActionGroup(groupId));
 724         List<InstructionInfo> instructions = new ArrayList<>();
 725         instructions.add(new InstructionApplyActions(actionsInfo));
 726         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 727         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 728                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 729                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 730
 731         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 732         return flowEntity;
 733     }
 734
 735     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 736
 737         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 738             routerName);
 739         List<MatchInfo> matches = new ArrayList<>();
 740         matches.add(MatchEthernetType.IPV4);
 741         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 742
 743         List<InstructionInfo> instructions = new ArrayList<>();
 744         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 745
 746         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 747         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 748             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 749             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 750
 751         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 752         return flowEntity;
 753     }
 754
 755     // TODO : Replace this with ITM Rpc once its available with full functionality
 756     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 757                                                      WriteTransaction writeFlowInvTx) {
 758         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 759             dpnId, routerName);
 760         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 761         if (flowEntity == null) {
 762             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 763                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 764                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 765                     dpnId, routerName);
 766             return;
 767         }
 768         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 769
 770     }
 771
 772     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 773         List<MatchInfo> matches = new ArrayList<>();
 774         matches.add(MatchEthernetType.IPV4);
 775         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 776                 routerName);
 777         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 778         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 779         List<InstructionInfo> instructions = new ArrayList<>();
 780         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 781                 MetaDataUtil.METADATA_MASK_VRFID));
 782         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 783         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 784                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 785                 instructions);
 786         return flowEntity;
 787     }
 788
 789     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 790         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 791                 .FLOWID_SEPARATOR + routerID;
 792     }
 793
 794     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 795         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 796                 .FLOWID_SEPARATOR + routerID;
 797     }
 798
 799     private String getGroupIdKey(String routerName) {
 800         return "snatmiss." + routerName;
 801     }
 802
 803     protected long createGroupId(String groupIdKey) {
 804         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 805             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 806             .build();
 807         try {
 808             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 809             RpcResult<AllocateIdOutput> rpcResult = result.get();
 810             return rpcResult.getResult().getIdValue();
 811         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 812             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 813         }
 814         return 0;
 815     }
 816
 817     protected void createGroupIdPool() {
 818         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 819             .setPoolName(NatConstants.SNAT_IDPOOL_NAME)
 820             .setLow(NatConstants.SNAT_ID_LOW_VALUE)
 821             .setHigh(NatConstants.SNAT_ID_HIGH_VALUE)
 822             .build();
 823         try {
 824             Future<RpcResult<Void>> result = idManager.createIdPool(createPool);
 825             if (result != null && result.get().isSuccessful()) {
 826                 LOG.debug("createGroupIdPool : GroupIdPool created successfully");
 827             } else {
 828                 LOG.error("createGroupIdPool : Unable to create GroupIdPool");
 829             }
 830         } catch (InterruptedException | ExecutionException e) {
 831             LOG.error("createGroupIdPool : Failed to create PortPool for NAPT Service", e);
 832         }
 833     }
 834
 835     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 836         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 837         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 838         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 839         List<BucketInfo> listBucketInfo = new ArrayList<>();
 840
 841         if (ifNamePrimary != null) {
 842             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 843             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(interfaceManager, ifNamePrimary, routerId);
 844             if (listActionInfoPrimary.isEmpty()) {
 845                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 846                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 847                         ifNamePrimary);
 848             }
 849         } else {
 850             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 851                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 852         }
 853         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 854
 855         listBucketInfo.add(0, bucketPrimary);
 856         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 857     }
 858
 859     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 860                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 861         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 862         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 863         List<BucketInfo> listBucketInfo = new ArrayList<>();
 864
 865         if (ifNamePrimary != null) {
 866             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 867                     ifNamePrimary);
 868             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(interfaceManager, ifNamePrimary, routerId);
 869             if (listActionInfoPrimary.isEmpty()) {
 870                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 871                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 872                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 873             }
 874         } else {
 875             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 876                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 877         }
 878         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 879
 880         listBucketInfo.add(0, bucketPrimary);
 881         return listBucketInfo;
 882     }
 883
 884     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 885             WriteTransaction writeFlowInvTx) {
 886        /*
 887         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 888         */
 889
 890         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 891
 892 /*
 893         List<BucketInfo> listBucketInfo = new ArrayList<>();
 894         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 895         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 896         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 897         listBucketInfo.add(0, bucketPrimary);
 898 */
 899
 900         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, writeFlowInvTx);
 901         installTerminatingServiceTblEntry(dpnId, routerName, routerId, writeFlowInvTx);
 902         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 903         installNaptPfibEntry(dpnId, routerId, writeFlowInvTx);
 904         Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 905         installNaptPfibEntriesForExternalSubnets(routerName, dpnId, writeFlowInvTx);
 906         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
 907         if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 908             installNaptPfibEntry(dpnId, vpnId, writeFlowInvTx);
 909         }
 910     }
 911
 912     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 913         List<BucketInfo> listBucketInfo = new ArrayList<>();
 914         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 915         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 916         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 917         listBucketInfo.add(0, bucketPrimary);
 918         return listBucketInfo;
 919     }
 920
 921     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, WriteTransaction writeFlowInvTx) {
 922         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 923         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 924         mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
 925     }
 926
 927     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 928
 929         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 930         List<MatchInfo> matches = new ArrayList<>();
 931         matches.add(MatchEthernetType.IPV4);
 932         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 933
 934         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 935         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 936         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
 937         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 938         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 939
 940         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
 941         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
 942             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 943             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
 944         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
 945         return flowEntity;
 946     }
 947
 948     public void handleSnatReverseTraffic(BigInteger dpnId, Routers router, long routerId, String routerName,
 949             String externalIp, WriteTransaction writeFlowInvTx) {
 950         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
 951             dpnId, routerId, externalIp);
 952         Uuid networkId = router.getNetworkId();
 953         if (networkId == null) {
 954             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
 955             return;
 956         }
 957         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
 958         if (vpnName == null) {
 959             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
 960                     + "configuration {} in router {}", networkId, externalIp, routerId);
 961             return;
 962         }
 963         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
 964             externalIp, networkId, router, writeFlowInvTx);
 965         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
 966             dpnId, routerId, externalIp);
 967     }
 968
 969     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
 970                                                 final long routerId, final String routerName, final String externalIp,
 971                                                 final Uuid extNetworkId, final Routers router,
 972                                                 final WriteTransaction writeFlowInvTx) {
 973         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
 974                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
 975         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 976         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 977         if (rd == null || rd.isEmpty()) {
 978             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
 979             return;
 980         }
 981         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
 982         if (extNwProvType == null) {
 983             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
 984             return;
 985         }
 986         if (extNwProvType == ProviderTypes.VXLAN) {
 987             WriteTransaction writeTx = dataBroker.newWriteOnlyTransaction();
 988             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
 989                     nextHopIp, writeTx, routerId, routerName, writeFlowInvTx);
 990             return;
 991         }
 992         //Generate VPN label for the external IP
 993         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 994             .setIpPrefix(externalIp).build();
 995         Future<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 996
 997         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
 998         ListenableFuture<RpcResult<Void>> future =
 999             Futures.transformAsync(JdkFutureAdapters.listenInPoolThread(labelFuture),
1000                 (AsyncFunction<RpcResult<GenerateVpnLabelOutput>, RpcResult<Void>>) result -> {
1001                     if (result.isSuccessful()) {
1002                         LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1003                         GenerateVpnLabelOutput output = result.getResult();
1004                         final long label = output.getLabel();
1005
1006                         int externalIpInDsFlag = 0;
1007                         //Get IPMaps from the DB for the router ID
1008                         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1009                         if (dbIpMaps != null) {
1010                             for (IpMap dbIpMap : dbIpMaps) {
1011                                 String dbExternalIp = dbIpMap.getExternalIp();
1012                                 //Select the IPMap, whose external IP is the IP for which FIB is installed
1013                                 if (dbExternalIp.contains(externalIp)) {
1014                                     String dbInternalIp = dbIpMap.getInternalIp();
1015                                     IpMapKey dbIpMapKey = dbIpMap.getKey();
1016                                     LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1017                                             + "and externalIp {}", label, dbInternalIp, externalIp);
1018                                     IpMap newIpm = new IpMapBuilder().setKey(dbIpMapKey).setInternalIp(dbInternalIp)
1019                                         .setExternalIp(dbExternalIp).setLabel(label).build();
1020                                     MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1021                                         naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1022                                     externalIpInDsFlag++;
1023                                 }
1024                             }
1025                             if (externalIpInDsFlag <= 0) {
1026                                 LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1027                                         + "Failed to update label {} for routerId {} in DS",
1028                                         externalIp, label, routerId);
1029                                 String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1030                                     + " found in DS for router %s", label, externalIp, routerId);
1031                                 return Futures.immediateFailedFuture(new Exception(errMsg));
1032                             }
1033                         } else {
1034                             LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1035                                     + " routerId {} in DS", label, externalIp, routerId);
1036                         }
1037                         //Inform BGP
1038                         long l3vni = 0;
1039                         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1040                             l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1041                         }
1042                         Routers extRouter = router != null ? router :
1043                             NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1044                         Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1045                                 extRouter);
1046                         NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1047                             externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1048                             RouteOrigin.STATIC, dpnId);
1049
1050                         //Install custom FIB routes
1051                         List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1052                         tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1053                         makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, writeFlowInvTx,
1054                                 extNwProvType);
1055                         makeLFibTableEntry(dpnId, label, tableId, writeFlowInvTx);
1056
1057                         //Install custom FIB routes - FIB table.
1058                         List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1059                                 routerName, externalIp);
1060                         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1061                             //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1062                             NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId,
1063                                     NwConstants.INBOUND_NAPT_TABLE,writeFlowInvTx);
1064                         }
1065                         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1066                         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1067                                 externalSubnetId);
1068                         String externalVpn = vpnName;
1069                         if (externalSubnet.isPresent()) {
1070                             externalVpn =  externalSubnetId.getValue();
1071                         }
1072                         CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1073                             .setVpnName(externalVpn)
1074                             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1075                             .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1076                             .setInstruction(fibTableCustomInstructions).build();
1077                         Future<RpcResult<Void>> future1 = fibService.createFibEntry(input);
1078                         return JdkFutureAdapters.listenInPoolThread(future1);
1079                     } else {
1080                         LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1081                         String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1082                             externalIp, vpnName, result.getErrors());
1083                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1084                     }
1085                 }, MoreExecutors.directExecutor());
1086
1087         Futures.addCallback(future, new FutureCallback<RpcResult<Void>>() {
1088
1089             @Override
1090             public void onFailure(@Nonnull Throwable error) {
1091                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1092             }
1093
1094             @Override
1095             public void onSuccess(@Nonnull RpcResult<Void> result) {
1096                 if (result.isSuccessful()) {
1097                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1098                             externalIp);
1099                 } else {
1100                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1101                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1102                 }
1103             }
1104         }, MoreExecutors.directExecutor());
1105     }
1106
1107     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1108             String externalIp) {
1109         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1110         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1111         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1112                 externalIp, router);
1113         int instructionIndex = 0;
1114         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1115             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1116             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1117                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1118             instructionIndex++;
1119         }
1120
1121         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1122         return fibTableCustomInstructions;
1123     }
1124
1125     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId, WriteTransaction writeFlowInvTx) {
1126         List<MatchInfo> matches = new ArrayList<>();
1127         matches.add(MatchEthernetType.MPLS_UNICAST);
1128         matches.add(new MatchMplsLabel(serviceId));
1129
1130         List<Instruction> instructions = new ArrayList<>();
1131         List<ActionInfo> actionsInfos = new ArrayList<>();
1132         actionsInfos.add(new ActionPopMpls());
1133         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1134         instructions.add(writeInstruction);
1135         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1136
1137         // Install the flow entry in L3_LFIB_TABLE
1138         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1139
1140         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1141             10, flowRef, 0, 0,
1142             COOKIE_VM_LFIB_TABLE, matches, instructions);
1143
1144         mdsalManager.addFlowToTx(dpId, flowEntity, writeFlowInvTx);
1145
1146         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1147     }
1148
1149     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1150              List<Instruction> customInstructions, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
1151         List<MatchInfo> mkMatches = new ArrayList<>();
1152
1153         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1154             dpnId, serviceId, customInstructions);
1155
1156         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1157             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1158         } else {
1159             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1160         }
1161
1162         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1163             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1164             String.format("%s:%d", "TST Flow Entry ", serviceId),
1165             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1166
1167         mdsalManager.addFlowToTx(dpnId, terminatingServiceTableFlowEntity, writeFlowInvTx);
1168     }
1169
1170     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1171         InstanceIdentifier<RouterIds> id = InstanceIdentifier.builder(
1172             RouterIdName.class).child(RouterIds.class, new RouterIdsKey(routerId)).build();
1173         return id;
1174     }
1175
1176     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1177         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1178                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1179     }
1180
1181     @Override
1182     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1183         String routerName = original.getRouterName();
1184         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1185         if (routerId == NatConstants.INVALID_ID) {
1186             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1187             return;
1188         }
1189         // Check if its update on SNAT flag
1190         boolean originalSNATEnabled = original.isEnableSnat();
1191         boolean updatedSNATEnabled = update.isEnableSnat();
1192         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1193             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1194         if (natMode == NatMode.Conntrack && !upgradeState.isUpgradeInProgress()) {
1195             if (originalSNATEnabled != updatedSNATEnabled) {
1196                 BigInteger primarySwitchId;
1197                 if (originalSNATEnabled) {
1198                     //SNAT disabled for the router
1199                     centralizedSwitchScheduler.releaseCentralizedSwitch(update);
1200                 } else {
1201                     centralizedSwitchScheduler.scheduleCentralizedSwitch(update);
1202                 }
1203             } else if (updatedSNATEnabled) {
1204                 centralizedSwitchScheduler.updateCentralizedSwitch(original,update);
1205             }
1206             List<ExternalIps> originalExternalIps = original.getExternalIps();
1207             List<ExternalIps> updateExternalIps = update.getExternalIps();
1208             if (!Objects.equals(originalExternalIps, updateExternalIps)) {
1209                 if (originalExternalIps == null || originalExternalIps.isEmpty()) {
1210                     centralizedSwitchScheduler.scheduleCentralizedSwitch(update);
1211                 }
1212             }
1213         } else {
1214             /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1215              * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1216              */
1217             long bgpVpnId = NatConstants.INVALID_ID;
1218             Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1219             if (bgpVpnUuid != null) {
1220                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1221             }
1222             BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1223             if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1224                 // Router has no interface attached
1225                 return;
1226             }
1227             final long finalBgpVpnId = bgpVpnId;
1228             coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.getKey(), () -> {
1229                 WriteTransaction writeFlowInvTx = dataBroker.newWriteOnlyTransaction();
1230                 WriteTransaction removeFlowInvTx = dataBroker.newWriteOnlyTransaction();
1231                 Uuid networkId = original.getNetworkId();
1232                 if (originalSNATEnabled != updatedSNATEnabled) {
1233                     if (originalSNATEnabled) {
1234                         //SNAT disabled for the router
1235                         Uuid networkUuid = original.getNetworkId();
1236                         LOG.info("update : SNAT disabled for Router {}", routerName);
1237                         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1238                         handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1239                                 removeFlowInvTx);
1240                     } else {
1241                         LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1242                         handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1243                     }
1244                 }
1245                 if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1246                     NatUtil.installRouterGwFlows(dataBroker, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1247                     NatUtil.installRouterGwFlows(dataBroker, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1248                 }
1249
1250                 //Check if the Update is on External IPs
1251                 LOG.debug("update : Checking if this is update on External IPs");
1252                 List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1253                 List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1254
1255                 //Check if the External IPs are added during the update.
1256                 Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1257                 addedExternalIps.removeAll(originalExternalIps);
1258                 if (addedExternalIps.size() != 0) {
1259                     LOG.debug("update : Start processing of the External IPs addition during the update operation");
1260                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1261                             update.getExtGwMacAddress(), dpnId,
1262                             update.getNetworkId(), null);
1263
1264                     for (String addedExternalIp : addedExternalIps) {
1265                     /*
1266                         1) Do nothing in the IntExtIp model.
1267                         2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1268                      */
1269                         String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1270                         String externalIp = externalIpParts[0];
1271                         String externalIpPrefix = externalIpParts[1];
1272                         String externalpStr = externalIp + "/" + externalIpPrefix;
1273                         LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1274                                         + "router ID {} in the ExternalIpsCounter model.",
1275                                 externalpStr, routerId);
1276                         naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1277                     }
1278                     LOG.debug("update : End processing of the External IPs addition during the update operation");
1279                 }
1280
1281                 //Check if the External IPs are removed during the update.
1282                 Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1283                 removedExternalIps.removeAll(updatedExternalIps);
1284                 if (removedExternalIps.size() > 0) {
1285                     LOG.debug("update : Start processing of the External IPs removal during the update operation");
1286                     vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1287                             removedExternalIps, original.getExtGwMacAddress(),
1288                             dpnId, networkId);
1289
1290                     for (String removedExternalIp : removedExternalIps) {
1291                     /*
1292                         1) Remove the mappings in the IntExt IP model which has external IP.
1293                         2) Remove the external IP in the ExternalCounter model.
1294                         3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1295                            least loaded external IP.
1296                            Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1297                         4) Increase the count of the allocated external IP by one.
1298                         5) Advertise to the BGP if external IP is allocated for the first time for the router
1299                          i.e. the route for the external IP is absent.
1300                         6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1301                          the removed external IPs and also from the model.
1302                         7) Advertise to the BGP for removing the route for the removed external IPs.
1303                      */
1304
1305                         String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1306                         String externalIp = externalIpParts[0];
1307                         String externalIpPrefix = externalIpParts[1];
1308                         String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1309
1310                         LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1311                                 + "entries for removed external IP {}", externalIpAddrStr);
1312                         Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1313                         String vpnName = "";
1314                         if (vpnUuId != null) {
1315                             vpnName = vpnUuId.getValue();
1316                         }
1317                         clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1318                                 update.getExtGwMacAddress(), removeFlowInvTx);
1319
1320                         LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1321                         //Get the internal IPs which are associated to the removed external IPs
1322                         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1323                         List<String> removedInternalIps = new ArrayList<>();
1324                         for (IpMap ipMap : ipMaps) {
1325                             if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1326                                 removedInternalIps.add(ipMap.getInternalIp());
1327                             }
1328                         }
1329
1330                         LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1331                         for (String removedInternalIp : removedInternalIps) {
1332                             LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1333                                             + "router ID {} from the IntExtIP model",
1334                                     removedInternalIp, routerId);
1335                             naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1336                         }
1337
1338                         LOG.debug("update : Remove the count mapping of the external IP {} for the "
1339                                         + "router ID {} from the ExternalIpsCounter model.",
1340                                 externalIpAddrStr, routerId);
1341                         naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1342
1343                         LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1344                                 + "whose external IPs were removed.");
1345                         for (String removedInternalIp : removedInternalIps) {
1346                             allocateExternalIp(dpnId, update, routerId, routerName, networkId, removedInternalIp,
1347                                     writeFlowInvTx);
1348                         }
1349
1350                         LOG.debug("update : Remove the NAPT translation entries from "
1351                                 + "Inbound and Outbound NAPT tables for the removed external IPs.");
1352                         //Get the internalIP and internal Port which were associated to the removed external IP.
1353                         List<Integer> externalPorts = new ArrayList<>();
1354                         Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1355                         InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1356                                 .builder(IntextIpPortMap.class)
1357                                 .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1358                         Optional<IpPortMapping> ipPortMapping =
1359                                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1360                         if (ipPortMapping.isPresent()) {
1361                             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1362                                     .getIntextIpProtocolType();
1363                             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1364                                 ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1365                                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1366                                 for (IpPortMap ipPortMap : ipPortMaps) {
1367                                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1368                                     if (ipPortExternal.getIpAddress().equals(externalIp)) {
1369                                         externalPorts.add(ipPortExternal.getPortNum());
1370                                         List<String> removedInternalIpPorts = protoTypesIntIpPortsMap.get(protoType);
1371                                         if (removedInternalIpPorts != null) {
1372                                             removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1373                                             protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1374                                         } else {
1375                                             removedInternalIpPorts = new ArrayList<>();
1376                                             removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1377                                             protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1378                                         }
1379                                     }
1380                                 }
1381                             }
1382                         }
1383
1384                         //Remove the IP port map from the intext-ip-port-map model, which were containing
1385                         // the removed external IP.
1386                         Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts = protoTypesIntIpPortsMap
1387                                 .entrySet();
1388                         Map<String, List<String>> internalIpPortMap = new HashMap<>();
1389                         for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1390                             ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1391                             List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1392                             for (String removedInternalIpPort : removedInternalIpPorts) {
1393                                 // Remove the IP port map from the intext-ip-port-map model,
1394                                 // which were containing the removed external IP
1395                                 naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort, protocolType);
1396                                 //Remove the IP port incomint packer map.
1397                                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR
1398                                         + removedInternalIpPort);
1399                                 String[] removedInternalIpPortParts = removedInternalIpPort
1400                                         .split(NatConstants.COLON_SEPARATOR);
1401                                 if (removedInternalIpPortParts.length == 2) {
1402                                     String removedInternalIp = removedInternalIpPortParts[0];
1403                                     String removedInternalPort = removedInternalIpPortParts[1];
1404                                     List<String> removedInternalPortsList = internalIpPortMap.get(removedInternalPort);
1405                                     if (removedInternalPortsList != null) {
1406                                         removedInternalPortsList.add(removedInternalPort);
1407                                         internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1408                                     } else {
1409                                         removedInternalPortsList = new ArrayList<>();
1410                                         removedInternalPortsList.add(removedInternalPort);
1411                                         internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1412                                     }
1413                                 }
1414                             }
1415                         }
1416
1417                         // Delete the entry from SnatIntIpPortMap DS
1418                         Set<String> internalIps = internalIpPortMap.keySet();
1419                         for (String internalIp : internalIps) {
1420                             LOG.debug("update : Removing IpPort having the internal IP {} from the "
1421                                     + "model SnatIntIpPortMap", internalIp);
1422                             naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1423                         }
1424
1425                         naptManager.removeNaptPortPool(externalIp);
1426
1427                         LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for the "
1428                                 + "removed external IP {}", externalIp);
1429                         for (Integer externalPort : externalPorts) {
1430                             //Remove the NAPT translation entries from Inbound NAPT table
1431                             naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1432                                     routerId, externalIp, externalPort);
1433                         }
1434
1435                         Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1436                         for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1437                             String internalIp = internalIpPort.getKey();
1438                             LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables for "
1439                                     + "the removed internal IP {}", internalIp);
1440                             List<String> internalPorts = internalIpPort.getValue();
1441                             for (String internalPort : internalPorts) {
1442                                 //Remove the NAPT translation entries from Outbound NAPT table
1443                                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR
1444                                         + internalIp + NatConstants.COLON_SEPARATOR + internalPort);
1445                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1446                                         routerId, internalIp, Integer.parseInt(internalPort));
1447                             }
1448                         }
1449                     }
1450                     LOG.debug("update : End processing of the External IPs removal during the update operation");
1451                 }
1452
1453                 //Check if its Update on subnets
1454                 LOG.debug("update : Checking if this is update on subnets");
1455                 List<Uuid> originalSubnetIds = original.getSubnetIds();
1456                 List<Uuid> updatedSubnetIds = update.getSubnetIds();
1457                 Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1458                 addedSubnetIds.removeAll(originalSubnetIds);
1459
1460                 //Check if the Subnet IDs are added during the update.
1461                 if (addedSubnetIds.size() != 0) {
1462                     LOG.debug("update : Start processing of the Subnet IDs addition during the update operation");
1463                     for (Uuid addedSubnetId : addedSubnetIds) {
1464                     /*
1465                         1) Select the least loaded external IP for the subnet and store the mapping of the
1466                         subnet IP and the external IP in the IntExtIp model.
1467                         2) Increase the count of the selected external IP by one.
1468                         3) Advertise to the BGP if external IP is allocated for the first time for the
1469                         router i.e. the route for the external IP is absent.
1470                      */
1471                         String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1472                         if (subnetIp != null) {
1473                             allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1474                                     writeFlowInvTx);
1475                         }
1476                     }
1477                     LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1478                 }
1479
1480                 //Check if the Subnet IDs are removed during the update.
1481                 Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1482                 removedSubnetIds.removeAll(updatedSubnetIds);
1483                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1484                 if (removedSubnetIds.size() != 0) {
1485                     LOG.debug("update : Start processing of the Subnet IDs removal during the update operation");
1486                     for (Uuid removedSubnetId : removedSubnetIds) {
1487                         String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1488                         if (subnetAddr != null) {
1489                         /*
1490                             1) Remove the subnet IP and the external IP in the IntExtIp map
1491                             2) Decrease the count of the coresponding external IP by one.
1492                             3) Advertise to the BGP for removing the routes of the corresponding external
1493                             IP if its not allocated to any other internal IP.
1494                         */
1495
1496                             String externalIp =
1497                                     naptManager.getExternalIpAllocatedForSubnet(routerId, subnetAddr[0] + "/"
1498                                             + subnetAddr[1]);
1499                             if (externalIp == null) {
1500                                 LOG.error("update : No mapping found for router ID {} and internal IP {}",
1501                                         routerId, subnetAddr[0]);
1502                                 futures.add(NatUtil.waitForTransactionToComplete(writeFlowInvTx));
1503                                 futures.add(NatUtil.waitForTransactionToComplete(removeFlowInvTx));
1504                                 return futures;
1505                             }
1506
1507                             naptManager.updateCounter(routerId, externalIp, false);
1508                             // Traverse entire model of external-ip counter whether external ip is not
1509                             // used by any other internal ip in any router
1510                             if (!isExternalIpAllocated(externalIp)) {
1511                                 LOG.debug("update : external ip is not allocated to any other "
1512                                         + "internal IP so proceeding to remove routes");
1513                                 clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId, Collections.singleton(externalIp),
1514                                         null, update.getExtGwMacAddress(), removeFlowInvTx);
1515                                 LOG.debug("update : Successfully removed fib entries in switch {} for "
1516                                                 + "router {} with networkId {} and externalIp {}",
1517                                         dpnId, routerId, networkId, externalIp);
1518                             }
1519
1520                             LOG.debug("update : Remove the IP mapping for the router ID {} and "
1521                                     + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1522                             naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1523                         }
1524                     }
1525                     LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1526                 }
1527                 futures.add(NatUtil.waitForTransactionToComplete(writeFlowInvTx));
1528                 futures.add(NatUtil.waitForTransactionToComplete(removeFlowInvTx));
1529                 return futures;
1530             }, NatConstants.NAT_DJC_MAX_RETRIES);
1531         } //end of controller based SNAT
1532     }
1533
1534     private boolean isExternalIpAllocated(String externalIp) {
1535         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1536         Optional<ExternalIpsCounter> externalCountersData =
1537             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1538         if (externalCountersData.isPresent()) {
1539             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1540             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1541             for (ExternalCounters ext : externalCounters) {
1542                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1543                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1544                         if (externalIpCount.getCounter() != 0) {
1545                             return true;
1546                         }
1547                         break;
1548                     }
1549                 }
1550             }
1551         }
1552         return false;
1553     }
1554
1555     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1556             Uuid networkId, String subnetIp, WriteTransaction writeFlowInvTx) {
1557         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1558         try {
1559             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1560             if (address instanceof Inet6Address) {
1561                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1562                 return;
1563             }
1564         } catch (UnknownHostException e) {
1565             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1566             return;
1567         }
1568         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1569         if (leastLoadedExtIpAddr != null) {
1570             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1571             String leastLoadedExtIp = externalIpParts[0];
1572             String leastLoadedExtIpPrefix = externalIpParts[1];
1573             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1574             subnetIp = subnetIpParts[0];
1575             String subnetIpPrefix = subnetIpParts[1];
1576             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1577             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1578                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1579                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1580             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1581
1582
1583             // Check if external IP is already assigned a route. (i.e. External IP is previously
1584             // allocated to any of the subnets)
1585             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1586             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1587             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1588             if (label != null) {
1589                 //update
1590                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1591                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1592                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1593                     label, internalIp, leastLoadedExtIpAddrStr);
1594                 IpMap newIpm = new IpMapBuilder().setKey(ipMapKey).setInternalIp(internalIp)
1595                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1596                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1597                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1598                 return;
1599             }
1600
1601             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1602             // for the first time and hence not having a route.
1603             //Get the VPN Name using the network ID
1604             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1605             if (vpnName != null) {
1606                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1607                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1608                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1609                         + "added after gateway-set");
1610                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1611                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1612                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1613                         return;
1614                     }
1615                 }
1616                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1617                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1618                     writeFlowInvTx);
1619             }
1620         }
1621     }
1622
1623     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1624         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1625         for (IpMap ipMap : ipMaps) {
1626             if (ipMap.getExternalIp().equals(externalIp)) {
1627                 if (ipMap.getLabel() != null) {
1628                     return ipMap.getLabel();
1629                 }
1630             }
1631         }
1632         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1633         return null;
1634     }
1635
1636     @Override
1637     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1638         LOG.trace("remove : Router delete method");
1639         {
1640             /*
1641             ROUTER DELETE SCENARIO
1642             1) Get the router ID from the event.
1643             2) Build the cookie information from the router ID.
1644             3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1645             4) Build the flow with the cookie value.
1646             5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1647             6) Remove the flows from the other switches which points to the primary and secondary
1648              switches for the flows related the router ID.
1649             7) Get the list of external IP address maintained for the router ID.
1650             8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1651             9) Withdraw the corresponding routes from the BGP.
1652              */
1653
1654             if (identifier == null || router == null) {
1655                 LOG.error("remove : returning without processing since routers is null");
1656                 return;
1657             }
1658
1659             String routerName = router.getRouterName();
1660             if (natMode == NatMode.Conntrack) {
1661                 if (router.isEnableSnat()) {
1662                     centralizedSwitchScheduler.releaseCentralizedSwitch(router);
1663                 }
1664             } else {
1665                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.getKey(), () -> {
1666                     LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ", routerName);
1667                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1668                     Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1669                     if (routerId == NatConstants.INVALID_ID) {
1670                         LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1671                                 routerName);
1672                         return futures;
1673                     }
1674                     long bgpVpnId = NatConstants.INVALID_ID;
1675                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1676                     if (bgpVpnUuid != null) {
1677                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1678                     }
1679                     WriteTransaction removeFlowInvTx = dataBroker.newWriteOnlyTransaction();
1680                     addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false, removeFlowInvTx);
1681                     Uuid networkUuid = router.getNetworkId();
1682
1683                     BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1684                     if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1685                         // No NAPT switch for external router, probably because the router is not attached to any
1686                         // internal networks
1687                         LOG.debug("No NAPT switch for router {}, check if router is attached to any internal network",
1688                                 routerName);
1689                         return futures;
1690                     } else {
1691                         NatUtil.installRouterGwFlows(dataBroker, vpnManager, router, primarySwitchId,
1692                                 NwConstants.DEL_FLOW);
1693                         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1694                         handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1695                                 routerId, removeFlowInvTx);
1696                     }
1697                     NatOverVxlanUtil.releaseVNI(routerName, idManager);
1698                     futures.add(NatUtil.waitForTransactionToComplete(removeFlowInvTx));
1699                     return futures;
1700                 }, NatConstants.NAT_DJC_MAX_RETRIES);
1701             }
1702         }
1703     }
1704
1705     // TODO Clean up the exception handling
1706     @SuppressWarnings("checkstyle:IllegalCatch")
1707     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1708                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1709                                   long routerId, WriteTransaction removeFlowInvTx) {
1710         LOG.info("handleDisableSnat : Entry");
1711         String routerName = router.getRouterName();
1712         try {
1713             if (routerFlag) {
1714                 removeNaptSwitch(routerName);
1715             } else {
1716                 updateNaptSwitch(routerName, BigInteger.ZERO);
1717             }
1718
1719             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1720             naptManager.removeExternalCounter(routerId);
1721
1722             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1723             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1724                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1725                     + "router ID {} from RouterNaptSwitch model", routerId);
1726                 return;
1727             }
1728             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1729                     networkUuid);
1730             if (extNwProvType == null) {
1731                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1732                 return;
1733             }
1734             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1735             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1736                     externalSubnetList, removeFlowInvTx, extNwProvType);
1737             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1738             try {
1739                 String externalSubnetVpn = null;
1740                 for (Uuid externalSubnetId : externalSubnetList) {
1741                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1742                     // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1743                     if (externalSubnet.isPresent()) {
1744                         externalSubnetVpn =  externalSubnetId.getValue();
1745                         clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1746                                 router.getExtGwMacAddress(), removeFlowInvTx);
1747                     }
1748                 }
1749                 if (externalSubnetVpn == null) {
1750                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1751                             router.getExtGwMacAddress(), removeFlowInvTx);
1752                 }
1753             } catch (Exception ex) {
1754                 LOG.error("handleDisableSnat : Failed to remove fib entries for routerId {} in naptSwitchDpnId {}",
1755                     routerId, naptSwitchDpnId, ex);
1756             }
1757             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1758             // for the router ID.
1759             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1760                     + "router ID {} in the DS", routerId);
1761             naptManager.removeMapping(routerId);
1762         } catch (Exception ex) {
1763             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, ex);
1764         }
1765         LOG.info("handleDisableSnat : Exit");
1766     }
1767
1768     // TODO Clean up the exception handling
1769     @SuppressWarnings("checkstyle:IllegalCatch")
1770     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1771                                              @Nonnull Collection<String> externalIps,
1772                                              String vpnId, WriteTransaction writeFlowInvTx) {
1773         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1774                 + "with internet vpn {}", routerName, vpnId);
1775         try {
1776             BigInteger naptSwitchDpnId = null;
1777             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1778                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1779             Optional<RouterToNaptSwitch> rtrToNapt =
1780                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1781             if (rtrToNapt.isPresent()) {
1782                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1783             }
1784             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1785
1786             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1787                     writeFlowInvTx);
1788             try {
1789                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1790                 if (extGwMacAddress != null) {
1791                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1792                             + "External Router ID {}", extGwMacAddress, routerId);
1793                 } else {
1794                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1795                             + "External Router ID {}", routerId);
1796                     return;
1797                 }
1798                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1799                         writeFlowInvTx);
1800             } catch (Exception ex) {
1801                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1802                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1803             }
1804             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1805         } catch (Exception ex) {
1806             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1807                     + "with internet vpn {}", routerName, vpnId, ex);
1808         }
1809         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1810                 routerName, vpnId);
1811     }
1812
1813     // TODO Clean up the exception handling
1814     @SuppressWarnings("checkstyle:IllegalCatch")
1815     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1816         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().setKey(new RouterToNaptSwitchKey(routerName))
1817             .setPrimarySwitchId(naptSwitchId).build();
1818         try {
1819             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1820                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1821         } catch (Exception ex) {
1822             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1823                 naptSwitchId, routerName);
1824         }
1825         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1826             naptSwitchId, routerName);
1827     }
1828
1829     protected void removeNaptSwitch(String routerName) {
1830         // Remove router and switch from model
1831         InstanceIdentifier<RouterToNaptSwitch> id =
1832             InstanceIdentifier.builder(NaptSwitches.class)
1833                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1834         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1835         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1836         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1837         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1838     }
1839
1840     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1841                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1842                                                 @Nonnull Collection<String> externalIps,
1843                                                 Collection<Uuid> externalSubnetList,
1844                                                 WriteTransaction removeFlowInvTx, ProviderTypes extNwProvType) {
1845         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1846         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1847
1848         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1849         // traffic which comes from the  VMs of the NAPT switches)
1850         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1851         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1852
1853         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1854                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1855         mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
1856
1857         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1858         // traffic which comes from the VMs of the non NAPT switches)
1859         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1860                 routerName);
1861         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1862         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1863         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1864                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1865         mdsalManager.removeFlowToTx(tsNatFlowEntity, removeFlowInvTx);
1866
1867         //Remove the flow table 25->44 from NAPT Switch
1868         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1869             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
1870         }
1871
1872         //Remove the Outbound flow entry which forwards the packet to FIB Table
1873         String outboundNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
1874         FlowEntity outboundNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1875             outboundNatFlowRef);
1876
1877         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1878                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1879         mdsalManager.removeFlowToTx(outboundNatFlowEntity, removeFlowInvTx);
1880
1881         removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, removeFlowInvTx);
1882         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1883         // External Subnet Vpn Id.
1884         for (Uuid externalSubnetId : externalSubnetList) {
1885             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1886             if (subnetVpnId != -1) {
1887                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1888                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1889                         natPfibSubnetFlowRef);
1890                 mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1891                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1892                         + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1893                         subnetVpnId, dpnId);
1894             }
1895         }
1896
1897         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1898         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1899         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1900
1901         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1902                 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1903         mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1904
1905         // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1906         // - This does not work since ext-routers is deleted already - no network info
1907         //Get the VPN ID from the ExternalNetworks model
1908         long vpnId = -1;
1909         if (vpnName == null || vpnName.isEmpty()) {
1910             // ie called from router delete cases
1911             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1912             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1913             if (vpnUuid != null) {
1914                 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1915                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete or "
1916                         + "disableSNAT scenario", vpnId, networkId);
1917             }
1918         } else {
1919             // ie called from disassociate vpn case
1920             LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
1921                     vpnName);
1922             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
1923             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}", vpnId);
1924         }
1925
1926         if (vpnId != NatConstants.INVALID_ID) {
1927             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
1928             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
1929             FlowEntity natPfibVpnFlowEntity =
1930                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
1931             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the DPN ID {} "
1932                     + "and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
1933             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, removeFlowInvTx);
1934         }
1935
1936         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
1937         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
1938         if (ipPortMapping == null) {
1939             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
1940             return;
1941         }
1942
1943         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
1944         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1945             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1946             for (IpPortMap ipPortMap : ipPortMaps) {
1947                 String ipPortInternal = ipPortMap.getIpPortInternal();
1948                 String[] ipPortParts = ipPortInternal.split(":");
1949                 if (ipPortParts.length != 2) {
1950                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
1951                     return;
1952                 }
1953                 String internalIp = ipPortParts[0];
1954                 String internalPort = ipPortParts[1];
1955
1956                 //Build the flow for the outbound NAPT table
1957                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
1958                         + NatConstants.COLON_SEPARATOR + internalPort);
1959                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1960                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
1961                 FlowEntity outboundNaptFlowEntity =
1962                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
1963
1964                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
1965                         + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1966                 mdsalManager.removeFlowToTx(outboundNaptFlowEntity, removeFlowInvTx);
1967
1968                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1969                 String externalIp = ipPortExternal.getIpAddress();
1970                 int externalPort = ipPortExternal.getPortNum();
1971
1972                 //Build the flow for the inbound NAPT table
1973                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1974                     String.valueOf(routerId), externalIp, externalPort);
1975                 FlowEntity inboundNaptFlowEntity =
1976                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
1977
1978                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
1979                         + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
1980                 mdsalManager.removeFlowToTx(inboundNaptFlowEntity, removeFlowInvTx);
1981             }
1982         }
1983     }
1984
1985     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
1986                                                     @Nonnull Collection<String> externalIps,
1987                                                     WriteTransaction writeFlowInvTx) {
1988         long extVpnId = NatConstants.INVALID_ID;
1989         if (networkId != null) {
1990             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1991             if (vpnUuid != null) {
1992                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1993             } else {
1994                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
1995             }
1996         } else {
1997             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
1998             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
1999         }
2000         if (extVpnId == NatConstants.INVALID_ID) {
2001             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2002             extVpnId = routerId;
2003         }
2004         for (String ip : externalIps) {
2005             String extIp = removeMaskFromIp(ip);
2006             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2007             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2008                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2009                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2010             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2011             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2012         }
2013     }
2014
2015     private String removeMaskFromIp(String ip) {
2016         if (ip != null && !ip.trim().isEmpty()) {
2017             return ip.split("/")[0];
2018         }
2019         return ip;
2020     }
2021
2022     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2023                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2024                                                            WriteTransaction writeFlowInvTx) {
2025         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2026         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2027
2028         //Remove the NAPT PFIB TABLE entry
2029         long vpnId = -1;
2030         if (vpnName != null) {
2031             // ie called from disassociate vpn case
2032             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2033                     + "with vpnName {}", vpnName);
2034             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2035             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2036                     vpnId);
2037         }
2038
2039         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2040                 networkId, routerName, dpnId)) {
2041             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2042             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2043             FlowEntity natPfibVpnFlowEntity =
2044                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2045             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2046                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2047             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2048
2049             // Remove IP-PORT active NAPT entries and release port from IdManager
2050             // For the router ID get the internal IP , internal port and the corresponding
2051             // external IP and external Port.
2052             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2053             if (ipPortMapping == null) {
2054                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2055                 return;
2056             }
2057             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2058             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2059                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2060                 for (IpPortMap ipPortMap : ipPortMaps) {
2061                     String ipPortInternal = ipPortMap.getIpPortInternal();
2062                     String[] ipPortParts = ipPortInternal.split(":");
2063                     if (ipPortParts.length != 2) {
2064                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2065                                 + "and port");
2066                         return;
2067                     }
2068                     String internalIp = ipPortParts[0];
2069                     String internalPort = ipPortParts[1];
2070
2071                     //Build the flow for the outbound NAPT table
2072                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2073                             + NatConstants.COLON_SEPARATOR + internalPort);
2074                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2075                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2076                     FlowEntity outboundNaptFlowEntity =
2077                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2078
2079                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2080                             + "active switch with the DPN ID {} and router ID {}",
2081                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2082                     mdsalManager.removeFlowToTx(outboundNaptFlowEntity, writeFlowInvTx);
2083
2084                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2085                     String externalIp = ipPortExternal.getIpAddress();
2086                     int externalPort = ipPortExternal.getPortNum();
2087
2088                     //Build the flow for the inbound NAPT table
2089                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2090                         String.valueOf(routerId), externalIp, externalPort);
2091                     FlowEntity inboundNaptFlowEntity =
2092                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2093
2094                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2095                             + "active active switch with the DPN ID {} and router ID {}",
2096                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2097                     mdsalManager.removeFlowToTx(inboundNaptFlowEntity, writeFlowInvTx);
2098
2099                     // Finally release port from idmanager
2100                     String internalIpPort = internalIp + ":" + internalPort;
2101                     naptManager.removePortFromPool(internalIpPort, externalIp);
2102
2103                     //Remove sessions from models
2104                     naptManager.removeIpPortMappingForRouterID(routerId);
2105                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2106                 }
2107             }
2108         } else {
2109             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2110         }
2111     }
2112
2113     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2114             BigInteger naptSwitchDpnId, WriteTransaction removeFlowInvTx) {
2115         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2116
2117         // Remove the flows from the other switches which points to the primary and secondary switches
2118         // for the flows related the router ID.
2119         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2120         if (allSwitchList.isEmpty()) {
2121             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2122             return;
2123         }
2124         for (BigInteger dpnId : allSwitchList) {
2125             if (!naptSwitchDpnId.equals(dpnId)) {
2126                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2127
2128                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2129                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2130                 FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2131
2132                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2133                         + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2134                 mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
2135
2136                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2137                 long groupId = createGroupId(getGroupIdKey(routerName));
2138                 List<BucketInfo> listBucketInfo = new ArrayList<>();
2139                 GroupEntity preSnatGroupEntity =
2140                     MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, listBucketInfo);
2141
2142                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2143                         + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2144                 mdsalManager.removeGroup(preSnatGroupEntity);
2145
2146             }
2147         }
2148     }
2149
2150     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2151                                          @Nonnull Collection<String> externalIps, String vpnName,
2152                                          String extGwMacAddress, WriteTransaction removeFlowInvTx) {
2153         //Withdraw the corresponding routes from the BGP.
2154         //Get the network ID using the router ID.
2155         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2156                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2157         if (networkUuid == null) {
2158             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2159             return;
2160         }
2161
2162         if (externalIps.isEmpty()) {
2163             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2164             return;
2165         }
2166
2167         if (vpnName == null) {
2168             //Get the VPN Name using the network ID
2169             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2170             if (vpnName == null) {
2171                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2172                     networkUuid, routerId);
2173                 return;
2174             }
2175         }
2176         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2177
2178         //Remove custom FIB routes
2179         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2180         for (String extIp : externalIps) {
2181             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, removeFlowInvTx);
2182         }
2183     }
2184
2185     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2186                                             final Uuid networkUuid, String extGwMacAddress,
2187                                             WriteTransaction removeFlowInvTx) {
2188         clearBgpRoutes(extIp, vpnName);
2189         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2190                 removeFlowInvTx);
2191     }
2192
2193     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2194                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2195                                              String gwMacAddress, boolean switchOver,
2196                                              WriteTransaction removeFlowInvTx) {
2197         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2198         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2199         if (routerName == null) {
2200             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2201             return;
2202         }
2203         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2204         if (extNwProvType == null) {
2205             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2206             return;
2207         }
2208         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2209          * external network provided type is VxLAN
2210          */
2211         if (extNwProvType == ProviderTypes.VXLAN) {
2212             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress,
2213                     removeFlowInvTx);
2214             return;
2215         }
2216         if (tempLabel < 0) {
2217             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2218                     extIp, routerId);
2219             return;
2220         }
2221
2222         final long label = tempLabel;
2223         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2224         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2225                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2226                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2227         Future<RpcResult<Void>> future = fibService.removeFibEntry(input);
2228
2229         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2230         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2231         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2232             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2233             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2234         }
2235         if (!switchOver) {
2236             ListenableFuture<RpcResult<Void>> labelFuture =
2237                     Futures.transformAsync(JdkFutureAdapters.listenInPoolThread(future),
2238                             (AsyncFunction<RpcResult<Void>, RpcResult<Void>>) result -> {
2239                                 //Release label
2240                             if (result.isSuccessful()) {
2241                                 NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2242                                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2243                                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
2244                                 Future<RpcResult<Void>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
2245                                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2246                             } else {
2247                                 String errMsg =
2248                                         String.format("RPC call to remove custom FIB entries on dpn %s for "
2249                                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2250                                 LOG.error(errMsg);
2251                                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2252                             }
2253                         });
2254
2255             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<Void>>() {
2256
2257                 @Override
2258                 public void onFailure(@Nonnull Throwable error) {
2259                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2260                         + "got external ip {}", label, extIp, error);
2261                 }
2262
2263                 @Override
2264                 public void onSuccess(@Nonnull RpcResult<Void> result) {
2265                     if (result.isSuccessful()) {
2266                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2267                             + "from VPN {}", externalIp, vpnName);
2268                     } else {
2269                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2270                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2271                     }
2272                 }
2273             }, MoreExecutors.directExecutor());
2274         } else {
2275             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2276                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2277         }
2278     }
2279
2280     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2281                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2282                                            WriteTransaction removeFlowInvTx) {
2283         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2284         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2285         if (routerName == null) {
2286             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2287             return;
2288         }
2289         //Get the external network provider type from networkId
2290         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2291         if (extNwProvType == null) {
2292             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2293                     networkUuid);
2294             return;
2295         }
2296         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2297          *  external network provided type is VxLAN
2298          */
2299         if (extNwProvType == ProviderTypes.VXLAN) {
2300             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress,
2301                     removeFlowInvTx);
2302             return;
2303         }
2304         //Get IPMaps from the DB for the router ID
2305         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2306         if (dbIpMaps.isEmpty()) {
2307             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2308             return;
2309         }
2310
2311         long tempLabel = NatConstants.INVALID_ID;
2312         for (IpMap dbIpMap : dbIpMaps) {
2313             String dbExternalIp = dbIpMap.getExternalIp();
2314             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2315             //Select the IPMap, whose external IP is the IP for which FIB is installed
2316             if (extIp.equals(dbExternalIp)) {
2317                 tempLabel = dbIpMap.getLabel();
2318                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2319                     tempLabel, dbExternalIp, routerId);
2320                 break;
2321             }
2322         }
2323         if (tempLabel == NatConstants.INVALID_ID) {
2324             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2325                     extIp, routerId);
2326             return;
2327         }
2328
2329         final long label = tempLabel;
2330         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2331         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2332                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2333                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2334         Future<RpcResult<Void>> future = fibService.removeFibEntry(input);
2335
2336         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2337         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2338         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2339             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2340             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2341         }
2342         if (!switchOver) {
2343             ListenableFuture<RpcResult<Void>> labelFuture =
2344                     Futures.transformAsync(JdkFutureAdapters.listenInPoolThread(future),
2345                             (AsyncFunction<RpcResult<Void>, RpcResult<Void>>) result -> {
2346                                 //Release label
2347                             if (result.isSuccessful()) {
2348                                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2349                                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
2350                                 Future<RpcResult<Void>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
2351                                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2352                             } else {
2353                                 String errMsg =
2354                                         String.format("RPC call to remove custom FIB entries on dpn %s for "
2355                                                         + "prefix %s Failed - %s",
2356                                                 dpnId, externalIp, result.getErrors());
2357                                 LOG.error(errMsg);
2358                                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2359                             }
2360                         });
2361
2362             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<Void>>() {
2363
2364                 @Override
2365                 public void onFailure(@Nonnull Throwable error) {
2366                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2367                 }
2368
2369                 @Override
2370                 public void onSuccess(@Nonnull RpcResult<Void> result) {
2371                     if (result.isSuccessful()) {
2372                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2373                             + "from VPN {}", externalIp, vpnName);
2374                     } else {
2375                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2376                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2377                     }
2378                 }
2379             }, MoreExecutors.directExecutor());
2380         } else {
2381             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2382                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2383         }
2384     }
2385
2386     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2387                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2388                                                WriteTransaction writeFlowInvTx) {
2389         //Withdraw the corresponding routes from the BGP.
2390         //Get the network ID using the router ID.
2391         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2392                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2393         if (networkUuid == null) {
2394             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2395             return;
2396         }
2397
2398         if (externalIps == null || externalIps.isEmpty()) {
2399             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2400             return;
2401         }
2402
2403         if (vpnName == null) {
2404             //Get the VPN Name using the network ID
2405             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2406             if (vpnName == null) {
2407                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2408                     networkUuid, routerId);
2409                 return;
2410             }
2411         }
2412         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2413
2414         //Remove custom FIB routes
2415         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2416         for (String extIp : externalIps) {
2417             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2418                     writeFlowInvTx);
2419         }
2420     }
2421
2422     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2423         //Inform BGP about the route removal
2424         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2425         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2426         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2427     }
2428
2429     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2430         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2431         List<MatchInfo> mkMatches = new ArrayList<>();
2432         // Matching metadata
2433         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
2434         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
2435             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
2436             5, String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
2437             COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
2438         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2439         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2440     }
2441
2442     private void removeLFibTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2443         List<MatchInfo> matches = new ArrayList<>();
2444         matches.add(MatchEthernetType.MPLS_UNICAST);
2445         matches.add(new MatchMplsLabel(serviceId));
2446
2447         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2448
2449         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2450
2451         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
2452             10, flowRef, 0, 0,
2453             COOKIE_VM_LFIB_TABLE, matches, null);
2454
2455         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2456
2457         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2458     }
2459
2460     /**
2461      * router association to vpn.
2462      *
2463      * @param routerName - Name of router
2464      * @param routerId - router id
2465      * @param bgpVpnName BGP VPN name
2466      */
2467     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2468             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2469         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2470         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2471             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2472
2473             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2474
2475             if (bgpVpnId != NatConstants.INVALID_ID) {
2476                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2477                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2478                 RouterIds rtrs = new RouterIdsBuilder().setKey(new RouterIdsKey(bgpVpnId))
2479                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2480                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2481                     getRoutersIdentifier(bgpVpnId), rtrs);
2482
2483                 // Get the allocated Primary NAPT Switch for this router
2484                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2485
2486                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2487                         routerId, bgpVpnId);
2488                 addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, true, writeFlowInvTx);
2489
2490                 // Get the group ID
2491                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2492                 createGroupId(getGroupIdKey(routerName));
2493                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2494                         extNwProvType);
2495             }
2496         }
2497     }
2498
2499     /**
2500      * router disassociation from vpn.
2501      *
2502      * @param routerName - Name of router
2503      * @param routerId - router id
2504      * @param bgpVpnName BGP VPN name
2505      */
2506     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2507             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2508         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2509         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2510             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2511             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2512
2513             // Get the allocated Primary NAPT Switch for this router
2514             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2515
2516             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2517             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID,
2518                     true, writeFlowInvTx);
2519
2520             // Get the group ID
2521             createGroupId(getGroupIdKey(routerName));
2522             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2523             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2524                     writeFlowInvTx, extNwProvType);
2525         }
2526     }
2527
2528     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2529         InstanceIdentifier<Routers> routerInstanceIndentifier =
2530             InstanceIdentifier.builder(ExtRouters.class)
2531                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2532         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2533         return routerData.isPresent() && routerData.get().isEnableSnat();
2534     }
2535
2536     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2537                                              long routerId, boolean isSnatCfgd, WriteTransaction writeFlowInvTx,
2538                                              ProviderTypes extNwProvType) {
2539         long changedVpnId = bgpVpnId;
2540         String idType = "BGP VPN";
2541         if (bgpVpnId == NatConstants.INVALID_ID) {
2542             changedVpnId = routerId;
2543             idType = "router";
2544         }
2545
2546         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2547         if (switches.isEmpty()) {
2548             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2549             return;
2550         }
2551         for (BigInteger dpnId : switches) {
2552             // Update the BGP VPN ID in the SNAT miss entry to group
2553             if (!dpnId.equals(primarySwitchId)) {
2554                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2555                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2556                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2557                 long groupId = createGroupId(getGroupIdKey(routerName));
2558                 if (!isSnatCfgd) {
2559                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2560                 }
2561
2562                 LOG.debug(
2563                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2564                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2565                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2566                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2567             } else {
2568                 LOG.debug(
2569                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2570                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2571                 FlowEntity flowEntity =
2572                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2573                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2574
2575                 LOG.debug(
2576                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2577                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2578                         idType, changedVpnId, primarySwitchId);
2579                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2580                         changedVpnId, writeFlowInvTx, extNwProvType);
2581
2582                 LOG.debug(
2583                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2584                                 + "which punts the packet to the controller in the Primary switch {}",
2585                         idType, changedVpnId, primarySwitchId);
2586                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2587
2588                 LOG.debug(
2589                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2590                                 + " outgoing packet to FIB Table in the Primary switch {}",
2591                         idType, changedVpnId, primarySwitchId);
2592                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2593
2594                 LOG.debug(
2595                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2596                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2597                                 + " {}", idType, changedVpnId, primarySwitchId);
2598                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2599
2600                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2601                         primarySwitchId);
2602                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2603                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2604                 if (vpnId != NatConstants.INVALID_ID) {
2605                     installNaptPfibEntry(primarySwitchId, vpnId, writeFlowInvTx);
2606                 }
2607             }
2608         }
2609     }
2610
2611     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2612         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2613         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2614         if (ipPortMapping == null) {
2615             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2616             return;
2617         }
2618         // Get the External Gateway MAC Address
2619         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2620         if (extGwMacAddress != null) {
2621             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2622                     extGwMacAddress, routerId);
2623         } else {
2624             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2625                     routerId);
2626             return;
2627         }
2628         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2629         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2630             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2631             for (IpPortMap ipPortMap : ipPortMaps) {
2632                 String ipPortInternal = ipPortMap.getIpPortInternal();
2633                 String[] ipPortParts = ipPortInternal.split(":");
2634                 if (ipPortParts.length != 2) {
2635                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2636                     return;
2637                 }
2638                 String internalIp = ipPortParts[0];
2639                 String internalPort = ipPortParts[1];
2640                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2641                         internalIp, internalPort);
2642                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2643                 NAPTEntryEvent.Protocol protocol;
2644                 switch (protocolTypes) {
2645                     case TCP:
2646                         protocol = NAPTEntryEvent.Protocol.TCP;
2647                         break;
2648                     case UDP:
2649                         protocol = NAPTEntryEvent.Protocol.UDP;
2650                         break;
2651                     default:
2652                         protocol = NAPTEntryEvent.Protocol.TCP;
2653                 }
2654                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2655                 SessionAddress externalAddress =
2656                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2657                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2658                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2659                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2660                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2661                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2662             }
2663         }
2664     }
2665
2666     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2667                                                           long changedVpnId) {
2668
2669         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2670             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2671         List<MatchInfo> matches = new ArrayList<>();
2672         matches.add(MatchEthernetType.IPV4);
2673         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2674
2675         List<ActionInfo> actionsInfo = new ArrayList<>();
2676         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2677                 routerName);
2678         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2679         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2680                 actionsInfo);
2681         actionsInfo.add(new ActionGroup(groupId));
2682         List<InstructionInfo> instructions = new ArrayList<>();
2683         instructions.add(new InstructionApplyActions(actionsInfo));
2684         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2685         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2686             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2687             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2688
2689         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2690         return flowEntity;
2691     }
2692
2693     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2694                                                                         long changedVpnId) {
2695
2696         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2697                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2698         List<MatchInfo> matches = new ArrayList<>();
2699         matches.add(MatchEthernetType.IPV4);
2700         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2701
2702         List<InstructionInfo> instructions = new ArrayList<>();
2703         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2704
2705         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2706         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2707             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2708             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2709
2710         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2711         return flowEntity;
2712     }
2713
2714     // TODO : Replace this with ITM Rpc once its available with full functionality
2715     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2716             long routerId, long changedVpnId, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2717         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2718             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2719         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2720                 extNwProvType);
2721         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2722     }
2723
2724     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2725             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2726         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2727             dpId, routerName, changedVpnId);
2728         List<MatchInfo> matches = new ArrayList<>();
2729         matches.add(MatchEthernetType.IPV4);
2730
2731         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2732         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2733             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2734         }
2735         matches.add(new MatchTunnelId(tunnelId));
2736
2737         List<InstructionInfo> instructions = new ArrayList<>();
2738         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2739             MetaDataUtil.METADATA_MASK_VRFID));
2740         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2741         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2742         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2743         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2744             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2745             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2746         return flowEntity;
2747     }
2748
2749     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2750                                                  WriteTransaction writeFlowInvTx) {
2751         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2752             dpnId, routerId, changedVpnId);
2753         FlowEntity flowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId);
2754         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing flow {}", flowEntity);
2755         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2756     }
2757
2758     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId, long changedVpnId) {
2759         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2760             dpId, routerId, changedVpnId);
2761         List<MatchInfo> matches = new ArrayList<>();
2762         matches.add(MatchEthernetType.IPV4);
2763         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2764
2765         List<InstructionInfo> instructions = new ArrayList<>();
2766         List<ActionInfo> actionsInfos = new ArrayList<>();
2767         actionsInfos.add(new ActionPuntToController());
2768         instructions.add(new InstructionApplyActions(actionsInfos));
2769         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2770             MetaDataUtil.METADATA_MASK_VRFID));
2771
2772         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId);
2773         BigInteger cookie = getCookieOutboundFlow(routerId);
2774         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2775             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2776         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2777         return flowEntity;
2778     }
2779
2780     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2781                                                WriteTransaction writeFlowInvTx) {
2782         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2783             dpnId, segmentId, changedVpnId);
2784         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2785         mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
2786     }
2787
2788     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2789
2790         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2791             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2792         List<MatchInfo> matches = new ArrayList<>();
2793         matches.add(MatchEthernetType.IPV4);
2794         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2795
2796         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2797         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2798         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2799         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2800         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2801
2802         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2803         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2804             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2805             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2806         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2807         return flowEntity;
2808     }
2809
2810     @Override
2811     protected ExternalRoutersListener getDataTreeChangeListener() {
2812         return ExternalRoutersListener.this;
2813     }
2814
2815     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2816                                                             WriteTransaction writeFlowInvTx) {
2817         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2818                 routerName);
2819         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2820             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2821             if (subnetVpnId != -1) {
2822                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2823                     + "and vpnId {}", dpnId, subnetVpnId);
2824                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2825             }
2826         }
2827     }
2828 }