natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.net.Inet6Address;
  21 import java.net.InetAddress;
  22 import java.net.UnknownHostException;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Collection;
  26 import java.util.Collections;
  27 import java.util.HashMap;
  28 import java.util.HashSet;
  29 import java.util.List;
  30 import java.util.Map;
  31 import java.util.Objects;
  32 import java.util.Set;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.Future;
  35 import javax.annotation.Nonnull;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.inject.Inject;
  39 import javax.inject.Singleton;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.UpgradeState;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  70 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  73 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  74 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  79 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  80 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  81 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  82 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  83 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  84 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  85 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  86 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  87 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 150 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 151 import org.opendaylight.yangtools.yang.binding.DataObject;
 152 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 153 import org.opendaylight.yangtools.yang.common.RpcResult;
 154 import org.slf4j.Logger;
 155 import org.slf4j.LoggerFactory;
 156
 157 @Singleton
 158 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 159     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 160
 161     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 162     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 163
 164     private final DataBroker dataBroker;
 165     private final ManagedNewTransactionRunner txRunner;
 166     private final IMdsalApiManager mdsalManager;
 167     private final ItmRpcService itmManager;
 168     private final OdlInterfaceRpcService odlInterfaceRpcService;
 169     private final IdManagerService idManager;
 170     private final NaptManager naptManager;
 171     private final NAPTSwitchSelector naptSwitchSelector;
 172     private final IBgpManager bgpManager;
 173     private final VpnRpcService vpnService;
 174     private final FibRpcService fibService;
 175     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 176     private final NaptEventHandler naptEventHandler;
 177     private final NaptPacketInHandler naptPacketInHandler;
 178     private final IFibManager fibManager;
 179     private final IVpnManager vpnManager;
 180     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 181     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 182     private final NatMode natMode;
 183     private final INeutronVpnManager nvpnManager;
 184     private final IElanService elanManager;
 185     private final JobCoordinator coordinator;
 186     private final UpgradeState upgradeState;
 187     private final IInterfaceManager interfaceManager;
 188     private final int snatPuntTimeout;
 189
 190     @Inject
 191     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 192                                    final ItmRpcService itmManager,
 193                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 194                                    final IdManagerService idManager,
 195                                    final NaptManager naptManager,
 196                                    final NAPTSwitchSelector naptSwitchSelector,
 197                                    final IBgpManager bgpManager,
 198                                    final VpnRpcService vpnService,
 199                                    final FibRpcService fibService,
 200                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 201                                    final NaptEventHandler naptEventHandler,
 202                                    final NaptPacketInHandler naptPacketInHandler,
 203                                    final IFibManager fibManager,
 204                                    final IVpnManager vpnManager,
 205                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 206                                    final INeutronVpnManager nvpnManager,
 207                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 208                                    final NatserviceConfig config,
 209                                    final IElanService elanManager,
 210                                    final JobCoordinator coordinator,
 211                                    final UpgradeState upgradeState,
 212                                    final IInterfaceManager interfaceManager) {
 213         super(Routers.class, ExternalRoutersListener.class);
 214         this.dataBroker = dataBroker;
 215         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 216         this.mdsalManager = mdsalManager;
 217         this.itmManager = itmManager;
 218         this.odlInterfaceRpcService = odlInterfaceRpcService;
 219         this.idManager = idManager;
 220         this.naptManager = naptManager;
 221         this.naptSwitchSelector = naptSwitchSelector;
 222         this.bgpManager = bgpManager;
 223         this.vpnService = vpnService;
 224         this.fibService = fibService;
 225         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 226         this.naptEventHandler = naptEventHandler;
 227         this.naptPacketInHandler = naptPacketInHandler;
 228         this.fibManager = fibManager;
 229         this.vpnManager = vpnManager;
 230         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 231         this.nvpnManager = nvpnManager;
 232         this.elanManager = elanManager;
 233         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 234         this.coordinator = coordinator;
 235         this.upgradeState = upgradeState;
 236         this.interfaceManager = interfaceManager;
 237         if (config != null) {
 238             this.natMode = config.getNatMode();
 239             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 240         } else {
 241             this.natMode = NatMode.Controller;
 242             this.snatPuntTimeout = 0;
 243         }
 244     }
 245
 246     @Override
 247     @PostConstruct
 248     public void init() {
 249         LOG.info("{} init", getClass().getSimpleName());
 250         // This class handles ExternalRouters for Controller SNAT mode.
 251         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 252         if (natMode == NatMode.Controller) {
 253             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 254             NatUtil.createGroupIdPool(idManager);
 255         }
 256     }
 257
 258     @Override
 259     protected InstanceIdentifier<Routers> getWildCardPath() {
 260         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 261     }
 262
 263     @Override
 264     // TODO Clean up the exception handling
 265     @SuppressWarnings("checkstyle:IllegalCatch")
 266     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 267         // Populate the router-id-name container
 268         String routerName = routers.getRouterName();
 269         LOG.info("add : external router event for {}", routerName);
 270         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 271         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 272         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 273         try {
 274             coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 275                 () -> Collections.singletonList(
 276                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 277                         LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 278                         long bgpVpnId = NatConstants.INVALID_ID;
 279                         if (bgpVpnUuid != null) {
 280                             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 281                         }
 282                         addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 283                         // Allocate Primary Napt Switch for this router
 284                         BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 285                         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 286                             if (!routers.isEnableSnat()) {
 287                                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 288                                 /* If SNAT is disabled on ext-router though L3_FIB_TABLE(21) -> PSNAT_TABLE(26) flow
 289                                  * is required for DNAT. Hence writeFlowInvTx object submit is required.
 290                                  */
 291                                 return;
 292                             }
 293                             handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 294                         }
 295                     })), NatConstants.NAT_DJC_MAX_RETRIES);
 296         } catch (Exception ex) {
 297             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 298                     routerName, ex);
 299         }
 300     }
 301
 302     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 303                                  TypedWriteTransaction<Configuration> confTx) {
 304         String routerName = routers.getRouterName();
 305         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 306
 307         naptManager.initialiseExternalCounter(routers, routerId);
 308         subnetRegisterMapping(routers, routerId);
 309
 310         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 311             primarySwitchId, routerName);
 312
 313         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 314                 routers.getNetworkId());
 315         if (extNwProvType == null) {
 316             LOG.error("handleEnableSnat : External Network Provider Type missing");
 317             return;
 318         }
 319
 320         if (bgpVpnId != NatConstants.INVALID_ID) {
 321             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 322                     extNwProvType);
 323         } else {
 324             // write metadata and punt
 325             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 326             // Now install entries in SNAT tables to point to Primary for each router
 327             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 328             for (BigInteger dpnId : switches) {
 329                 // Handle switches and NAPT switches separately
 330                 if (!dpnId.equals(primarySwitchId)) {
 331                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 332                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 333                 } else {
 334                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 335                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 336                 }
 337             }
 338         }
 339
 340         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 341         if (externalIps.isEmpty()) {
 342             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 343             return;
 344         } else {
 345             for (String externalIpAddrPrefix : externalIps) {
 346                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 347                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 348                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 349                 );
 350             }
 351         }
 352         LOG.debug("handleEnableSnat : Exit");
 353     }
 354
 355     private BigInteger getPrimaryNaptSwitch(String routerName) {
 356         // Allocate Primary Napt Switch for this router
 357         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 358         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 359             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 360                 primarySwitchId, routerName);
 361             return primarySwitchId;
 362         }
 363         // Validating and creating VNI pool during when NAPT switch is selected.
 364         // With Assumption this might be the first NAT service comes up.
 365         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 366                 NatConstants.ODL_VNI_POOL_NAME);
 367         // Allocated an id from VNI pool for the Router.
 368         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 369         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 370         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 371
 372         return primarySwitchId;
 373     }
 374
 375     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 376                                                      TypedWriteTransaction<Configuration> confTx) {
 377         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 378         if (extVpnId == NatConstants.INVALID_ID) {
 379             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 380             extVpnId = routerId;
 381         }
 382         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 383         if (externalIps.isEmpty()) {
 384             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 385                 dpnId, extVpnId);
 386             return;
 387         }
 388         for (String ip : externalIps) {
 389             Uuid subnetId = getSubnetIdForFixedIp(ip);
 390             if (subnetId != null) {
 391                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 392                 if (subnetVpnId != NatConstants.INVALID_ID) {
 393                     extVpnId = subnetVpnId;
 394                 }
 395                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 396                     dpnId, extVpnId, subnetId);
 397                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 398                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 399             }
 400         }
 401     }
 402
 403     @Nullable
 404     private Uuid getSubnetIdForFixedIp(String ip) {
 405         if (ip != null) {
 406             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 407             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 408             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 409         }
 410         LOG.error("getSubnetIdForFixedIp : ip is null");
 411         return null;
 412     }
 413
 414     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 415         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 416         List<Uuid> subnetList = routerEntry.getSubnetIds();
 417         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 418         int counter = 0;
 419         int extIpCounter = externalIps.size();
 420         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 421                 counter, extIpCounter);
 422         for (Uuid subnet : subnetList) {
 423             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 424             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 425                 .builder(Subnetmaps.class)
 426                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 427                 .build();
 428             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 429             if (sn.isPresent()) {
 430                 // subnets
 431                 Subnetmap subnetmapEntry = sn.get();
 432                 String subnetString = subnetmapEntry.getSubnetIp();
 433                 String[] subnetSplit = subnetString.split("/");
 434                 String subnetIp = subnetSplit[0];
 435                 try {
 436                     InetAddress address = InetAddress.getByName(subnetIp);
 437                     if (address instanceof Inet6Address) {
 438                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 439                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 440                         continue;
 441                     }
 442                 } catch (UnknownHostException e) {
 443                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 444                     return;
 445                 }
 446                 String subnetPrefix = "0";
 447                 if (subnetSplit.length == 2) {
 448                     subnetPrefix = subnetSplit[1];
 449                 }
 450                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 451                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 452                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 453                 //externalIps
 454                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 455                         counter, extIpCounter);
 456                 if (extIpCounter != 0) {
 457                     if (counter < extIpCounter) {
 458                         String[] ipSplit = externalIps.get(counter).split("/");
 459                         String externalIp = ipSplit[0];
 460                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 461                         if (ipSplit.length == 2) {
 462                             extPrefix = ipSplit[1];
 463                         }
 464                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 465                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 466                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 467                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 468                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 469                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 470                     } else {
 471                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 472                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 473                         String[] ipSplit = externalIps.get(counter).split("/");
 474                         String externalIp = ipSplit[0];
 475                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 476                         if (ipSplit.length == 2) {
 477                             extPrefix = ipSplit[1];
 478                         }
 479                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 480                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 481                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 482                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 483                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 484                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 485                             externalIp, extPrefix);
 486                     }
 487                 }
 488                 counter++;
 489                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 490             } else {
 491                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 492             }
 493         }
 494     }
 495
 496     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 497             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx) {
 498         //Check if BGP VPN exists. If exists then invoke the new method.
 499         if (bgpVpnId != NatConstants.INVALID_ID) {
 500             if (bgpVpnUuid != null) {
 501                 String bgpVpnName = bgpVpnUuid.getValue();
 502                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 503                     bgpVpnId, bgpVpnName);
 504                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 505                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 506                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 507             }
 508             if (create) {
 509                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 510             } else {
 511                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 512             }
 513             return;
 514         }
 515
 516         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 517         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 518     }
 519
 520     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 521             TypedReadWriteTransaction<Configuration> confTx) {
 522         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 523         if (switches.isEmpty()) {
 524             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 525             return;
 526         }
 527         if (routerId == NatConstants.INVALID_ID) {
 528             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 529                     + "default NAT route in FIB", routerName);
 530             return;
 531         }
 532         for (BigInteger dpnId : switches) {
 533             if (create) {
 534                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 535                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 536                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 537             } else {
 538                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 539                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 540                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 541             }
 542         }
 543     }
 544
 545     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 546         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 547         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 548         if (dpnIds.isEmpty()) {
 549             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 550                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 551             return;
 552         }
 553         for (BigInteger dpnId : dpnIds) {
 554             if (bgpVpnId != NatConstants.INVALID_ID) {
 555                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 556                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 557                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 558             } else {
 559                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 560                     + "in dpn {} for the internal vpn", routerId, dpnId);
 561                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 562             }
 563         }
 564     }
 565
 566     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 567         long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx) {
 568         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 569         if (dpnIds.isEmpty()) {
 570             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 571                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 572             return;
 573         }
 574         for (BigInteger dpnId : dpnIds) {
 575             if (bgpVpnId != NatConstants.INVALID_ID) {
 576                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 577                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 578                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 579             } else {
 580                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 581                     + "in dpn {} for the internal vpn", routerId, dpnId);
 582                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 583             }
 584         }
 585     }
 586
 587     // TODO Clean up the exception handling
 588     @SuppressWarnings("checkstyle:IllegalCatch")
 589     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 590                                                           InstanceIdentifier<T> path) {
 591         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 592
 593         try {
 594             return tx.read(datastoreType, path).get();
 595         } catch (Exception e) {
 596             throw new RuntimeException(e);
 597         }
 598     }
 599
 600     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 601         TypedWriteTransaction<Configuration> confTx) {
 602         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 603         if (routerId != NatConstants.INVALID_ID) {
 604             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 605                     primarySwitchId, routerId);
 606             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 607         } else {
 608             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 609                 + "createAndInstallMissEntry", routerName);
 610         }
 611     }
 612
 613     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 614         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 615                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 616     }
 617
 618     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 619         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 620                 .FLOWID_SEPARATOR + vpnId;
 621     }
 622
 623     public BigInteger getCookieOutboundFlow(long routerId) {
 624         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 625             BigInteger.valueOf(routerId));
 626     }
 627
 628     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 629         long l4SrcPortField;
 630         long l4DstPortField;
 631         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 632
 633         if (protocol == NwConstants.IP_PROT_TCP) {
 634             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 635             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 636         } else {
 637             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 638             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 639         }
 640         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 641                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 642                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 643                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 644                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 645                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 646                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 647                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 648                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 649                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 650                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 651                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 652                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 653                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 654                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 655
 656         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 657                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 658     }
 659
 660     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 661         List<MatchInfo> matches = new ArrayList<>();
 662         matches.add(MatchEthernetType.IPV4);
 663         matches.add(MatchIpProtocol.ICMP);
 664         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 665
 666         List<ActionInfo> actionInfos = new ArrayList<>();
 667         actionInfos.add(new ActionDrop());
 668
 669         List<InstructionInfo> instructions = new ArrayList<>();
 670         instructions.add(new InstructionApplyActions(actionInfos));
 671
 672         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 673                 NwConstants.IP_PROT_ICMP);
 674         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 675                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 676                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 677         return flow;
 678     }
 679
 680     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 681         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 682         BigInteger cookie = getCookieOutboundFlow(routerId);
 683         List<MatchInfo> matches = new ArrayList<>();
 684         matches.add(MatchEthernetType.IPV4);
 685         matches.add(new MatchIpProtocol((short)protocol));
 686         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 687
 688         List<InstructionInfo> instructions = new ArrayList<>();
 689         List<ActionInfo> actionsInfos = new ArrayList<>();
 690         actionsInfos.add(new ActionPuntToController());
 691         if (snatPuntTimeout != 0) {
 692             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 693         }
 694         instructions.add(new InstructionApplyActions(actionsInfos));
 695
 696         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 697         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 698             5, flowRef, 0, 0,
 699             cookie, matches, instructions);
 700         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 701         return flowEntity;
 702     }
 703
 704     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 705         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 706         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 707         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 708         mdsalManager.addFlow(confTx, tcpflowEntity);
 709
 710         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 711         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 712         mdsalManager.addFlow(confTx, udpflowEntity);
 713
 714         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 715         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 716         mdsalManager.addFlow(confTx, icmpDropFlow);
 717     }
 718
 719     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 720         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 721         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 722         try {
 723             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 724                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 725                     .setSourceDpid(srcDpId)
 726                     .setDestinationDpid(dstDpId)
 727                     .setTunnelType(tunType)
 728                     .build());
 729             rpcResult = result.get();
 730             if (!rpcResult.isSuccessful()) {
 731                 tunType = TunnelTypeGre.class;
 732                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 733                     .setSourceDpid(srcDpId)
 734                     .setDestinationDpid(dstDpId)
 735                     .setTunnelType(tunType)
 736                     .build());
 737                 rpcResult = result.get();
 738                 if (!rpcResult.isSuccessful()) {
 739                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 740                             rpcResult.getErrors());
 741                 } else {
 742                     return rpcResult.getResult().getInterfaceName();
 743                 }
 744                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 745                         rpcResult.getErrors());
 746             } else {
 747                 return rpcResult.getResult().getInterfaceName();
 748             }
 749         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 750             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 751                     + "between {} and {}", srcDpId, dstDpId, e);
 752         }
 753
 754         return null;
 755     }
 756
 757     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 758         TypedWriteTransaction<Configuration> confTx) {
 759
 760         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 761         // Install miss entry pointing to group
 762         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 763         mdsalManager.addFlow(confTx, flowEntity);
 764     }
 765
 766     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 767             String routerName, long routerId) {
 768         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 769             dpnId, bucketInfo.get(0));
 770         // Install the select group
 771         long groupId = createGroupId(getGroupIdKey(routerName));
 772         GroupEntity groupEntity =
 773             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 774         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 775         mdsalManager.syncInstallGroup(groupEntity);
 776         // Install miss entry pointing to group
 777         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 778         if (flowEntity == null) {
 779             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 780                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 781                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 782             return;
 783         }
 784         mdsalManager.installFlow(flowEntity);
 785     }
 786
 787     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 788         long groupId = createGroupId(getGroupIdKey(routerName));
 789         GroupEntity groupEntity =
 790             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 791         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 792         mdsalManager.syncInstallGroup(groupEntity);
 793         return groupId;
 794     }
 795
 796     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 797         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 798                 dpId, routerName, groupId);
 799         List<MatchInfo> matches = new ArrayList<>();
 800         matches.add(MatchEthernetType.IPV4);
 801         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 802
 803         List<ActionInfo> actionsInfo = new ArrayList<>();
 804         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 805                 routerName);
 806         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 807         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 808         actionsInfo.add(new ActionGroup(groupId));
 809         List<InstructionInfo> instructions = new ArrayList<>();
 810         instructions.add(new InstructionApplyActions(actionsInfo));
 811         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 812         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 813                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 814                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 815
 816         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 817         return flowEntity;
 818     }
 819
 820     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 821
 822         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 823             routerName);
 824         List<MatchInfo> matches = new ArrayList<>();
 825         matches.add(MatchEthernetType.IPV4);
 826         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 827
 828         List<InstructionInfo> instructions = new ArrayList<>();
 829         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 830
 831         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 832         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 833             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 834             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 835
 836         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 837         return flowEntity;
 838     }
 839
 840     // TODO : Replace this with ITM Rpc once its available with full functionality
 841     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 842         TypedWriteTransaction<Configuration> confTx) {
 843
 844         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 845             dpnId, routerName);
 846         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 847         if (flowEntity == null) {
 848             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 849                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 850                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 851                     dpnId, routerName);
 852             return;
 853         }
 854         mdsalManager.addFlow(confTx, flowEntity);
 855
 856     }
 857
 858     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 859         List<MatchInfo> matches = new ArrayList<>();
 860         matches.add(MatchEthernetType.IPV4);
 861         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 862                 routerName);
 863         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 864         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 865         List<InstructionInfo> instructions = new ArrayList<>();
 866         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 867                 MetaDataUtil.METADATA_MASK_VRFID));
 868         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 869         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 870                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 871                 instructions);
 872         return flowEntity;
 873     }
 874
 875     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 876         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 877                 .FLOWID_SEPARATOR + routerID;
 878     }
 879
 880     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 881         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 882                 .FLOWID_SEPARATOR + routerID;
 883     }
 884
 885     private String getGroupIdKey(String routerName) {
 886         return "snatmiss." + routerName;
 887     }
 888
 889     protected long createGroupId(String groupIdKey) {
 890         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 891             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 892             .build();
 893         try {
 894             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 895             RpcResult<AllocateIdOutput> rpcResult = result.get();
 896             return rpcResult.getResult().getIdValue();
 897         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 898             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 899         }
 900         return 0;
 901     }
 902
 903     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 904         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 905         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 906         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 907         List<BucketInfo> listBucketInfo = new ArrayList<>();
 908
 909         if (ifNamePrimary != null) {
 910             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 911             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 912                     interfaceManager, ifNamePrimary, routerId, true);
 913             if (listActionInfoPrimary.isEmpty()) {
 914                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 915                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 916                         ifNamePrimary);
 917             }
 918         } else {
 919             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 920                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 921         }
 922         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 923
 924         listBucketInfo.add(0, bucketPrimary);
 925         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 926     }
 927
 928     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 929                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 930         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 931         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 932         List<BucketInfo> listBucketInfo = new ArrayList<>();
 933
 934         if (ifNamePrimary != null) {
 935             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 936                     ifNamePrimary);
 937             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 938                     interfaceManager, ifNamePrimary, routerId, true);
 939             if (listActionInfoPrimary.isEmpty()) {
 940                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 941                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 942                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 943             }
 944         } else {
 945             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 946                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 947         }
 948         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 949
 950         listBucketInfo.add(0, bucketPrimary);
 951         return listBucketInfo;
 952     }
 953
 954     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 955         TypedWriteTransaction<Configuration> confTx) {
 956
 957        /*
 958         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 959         */
 960
 961         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 962
 963 /*
 964         List<BucketInfo> listBucketInfo = new ArrayList<>();
 965         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 966         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 967         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 968         listBucketInfo.add(0, bucketPrimary);
 969 */
 970
 971         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 972         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 973         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 974         installNaptPfibEntry(dpnId, routerId, confTx);
 975         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 976         if (networkId != null) {
 977             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 978             if (vpnUuid != null) {
 979                 Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 980                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 981                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 982                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 983                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 984                         installNaptPfibEntry(dpnId, vpnId, null);
 985                     }
 986                     return Collections.emptyList();
 987                 });
 988             } else {
 989                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 990             }
 991         } else {
 992             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 993         }
 994     }
 995
 996     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 997         List<BucketInfo> listBucketInfo = new ArrayList<>();
 998         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 999         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
1000         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
1001         listBucketInfo.add(0, bucketPrimary);
1002         return listBucketInfo;
1003     }
1004
1005     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, TypedWriteTransaction<Configuration> confTx) {
1006         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1007         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1008         if (confTx != null) {
1009             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1010         } else {
1011             mdsalManager.installFlow(naptPfibFlowEntity);
1012         }
1013     }
1014
1015     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1016
1017         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1018         List<MatchInfo> matches = new ArrayList<>();
1019         matches.add(MatchEthernetType.IPV4);
1020         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1021
1022         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1023         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1024         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1025         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1026         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1027
1028         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1029         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1030             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1031             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1032         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1033         return flowEntity;
1034     }
1035
1036     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1037         long routerId, String routerName, String externalIp) {
1038         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1039             dpnId, routerId, externalIp);
1040         Uuid networkId = router.getNetworkId();
1041         if (networkId == null) {
1042             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1043             return;
1044         }
1045         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1046         if (vpnName == null) {
1047             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1048                     + "configuration {} in router {}", networkId, externalIp, routerId);
1049             return;
1050         }
1051         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1052             externalIp, networkId, router, confTx);
1053         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1054             dpnId, routerId, externalIp);
1055     }
1056
1057     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1058                                                 final long routerId, final String routerName, final String externalIp,
1059                                                 final Uuid extNetworkId, final Routers router,
1060                                                 final TypedWriteTransaction<Configuration> confTx) {
1061         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1062                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1063         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1064         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1065         if (rd == null || rd.isEmpty()) {
1066             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1067             return;
1068         }
1069         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1070         if (extNwProvType == null) {
1071             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1072             return;
1073         }
1074         if (extNwProvType == ProviderTypes.VXLAN) {
1075             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1076                 nextHopIp, routerId, routerName, confTx);
1077             return;
1078         }
1079         //Generate VPN label for the external IP
1080         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1081             .setIpPrefix(externalIp).build();
1082         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1083
1084         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1085         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1086             if (result.isSuccessful()) {
1087                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1088                 GenerateVpnLabelOutput output = result.getResult();
1089                 final long label = output.getLabel();
1090
1091                 int externalIpInDsFlag = 0;
1092                 //Get IPMaps from the DB for the router ID
1093                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1094                 if (dbIpMaps != null) {
1095                     for (IpMap dbIpMap : dbIpMaps) {
1096                         String dbExternalIp = dbIpMap.getExternalIp();
1097                         //Select the IPMap, whose external IP is the IP for which FIB is installed
1098                         if (dbExternalIp.contains(externalIp)) {
1099                             String dbInternalIp = dbIpMap.getInternalIp();
1100                             IpMapKey dbIpMapKey = dbIpMap.key();
1101                             LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1102                                     + "and externalIp {}", label, dbInternalIp, externalIp);
1103                             IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1104                                 .setExternalIp(dbExternalIp).setLabel(label).build();
1105                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1106                                 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1107                             externalIpInDsFlag++;
1108                         }
1109                     }
1110                     if (externalIpInDsFlag <= 0) {
1111                         LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1112                                 + "Failed to update label {} for routerId {} in DS",
1113                                 externalIp, label, routerId);
1114                         String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1115                             + " found in DS for router %s", label, externalIp, routerId);
1116                         return Futures.immediateFailedFuture(new Exception(errMsg));
1117                     }
1118                 } else {
1119                     LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1120                             + " routerId {} in DS", label, externalIp, routerId);
1121                 }
1122                 //Inform BGP
1123                 long l3vni = 0;
1124                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1125                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1126                 }
1127                 Routers extRouter = router != null ? router :
1128                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1129                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1130                         extRouter);
1131                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1132                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1133                     RouteOrigin.STATIC, dpnId);
1134
1135                 //Install custom FIB routes
1136                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1137                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1138                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1139                         extNwProvType);
1140                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1141
1142                 //Install custom FIB routes - FIB table.
1143                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1144                         routerName, externalIp);
1145                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1146                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1147                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1148                 }
1149                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1150                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1151                         externalSubnetId);
1152                 String externalVpn = vpnName;
1153                 if (externalSubnet.isPresent()) {
1154                     externalVpn =  externalSubnetId.getValue();
1155                 }
1156                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1157                     .setVpnName(externalVpn)
1158                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1159                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1160                     .setInstruction(fibTableCustomInstructions).build();
1161                 return fibService.createFibEntry(input);
1162             } else {
1163                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1164                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1165                     externalIp, vpnName, result.getErrors());
1166                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1167             }
1168         }, MoreExecutors.directExecutor());
1169
1170         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1171
1172             @Override
1173             public void onFailure(@Nonnull Throwable error) {
1174                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1175             }
1176
1177             @Override
1178             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1179                 if (result.isSuccessful()) {
1180                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1181                             externalIp);
1182                 } else {
1183                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1184                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1185                 }
1186             }
1187         }, MoreExecutors.directExecutor());
1188     }
1189
1190     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1191             String externalIp) {
1192         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1193         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1194         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1195                 externalIp, router);
1196         int instructionIndex = 0;
1197         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1198             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1199             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1200                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1201             instructionIndex++;
1202         }
1203
1204         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1205         return fibTableCustomInstructions;
1206     }
1207
1208     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1209         TypedWriteTransaction<Configuration> confTx) {
1210         List<MatchInfo> matches = new ArrayList<>();
1211         matches.add(MatchEthernetType.MPLS_UNICAST);
1212         matches.add(new MatchMplsLabel(serviceId));
1213
1214         List<Instruction> instructions = new ArrayList<>();
1215         List<ActionInfo> actionsInfos = new ArrayList<>();
1216         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1217         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1218         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1219         instructions.add(writeInstruction);
1220         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1221
1222         // Install the flow entry in L3_LFIB_TABLE
1223         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1224
1225         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1226             10, flowRef, 0, 0,
1227             COOKIE_VM_LFIB_TABLE, matches, instructions);
1228
1229         mdsalManager.addFlow(confTx, dpId, flowEntity);
1230
1231         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1232     }
1233
1234     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1235         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1236         ProviderTypes extNwProvType) {
1237
1238         List<MatchInfo> mkMatches = new ArrayList<>();
1239
1240         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1241             dpnId, serviceId, customInstructions);
1242
1243         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1244             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1245         } else {
1246             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1247         }
1248
1249         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1250             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1251             String.format("%s:%d", "TST Flow Entry ", serviceId),
1252             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1253
1254         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1255     }
1256
1257     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1258         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1259             new RouterIdsKey(routerId)).build();
1260     }
1261
1262     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1263         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1264                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1265     }
1266
1267     @Override
1268     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1269         String routerName = original.getRouterName();
1270         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1271         if (routerId == NatConstants.INVALID_ID) {
1272             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1273             return;
1274         }
1275         // Check if its update on SNAT flag
1276         boolean originalSNATEnabled = original.isEnableSnat();
1277         boolean updatedSNATEnabled = update.isEnableSnat();
1278         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1279             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1280         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1281          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1282          */
1283         long bgpVpnId = NatConstants.INVALID_ID;
1284         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1285         if (bgpVpnUuid != null) {
1286             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1287         }
1288         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1289         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1290             // Router has no interface attached
1291             return;
1292         }
1293         final long finalBgpVpnId = bgpVpnId;
1294         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1295             List<ListenableFuture<Void>> futures = new ArrayList<>();
1296             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1297                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1298                     Uuid networkId = original.getNetworkId();
1299                     if (originalSNATEnabled != updatedSNATEnabled) {
1300                         if (originalSNATEnabled) {
1301                             //SNAT disabled for the router
1302                             Uuid networkUuid = original.getNetworkId();
1303                             LOG.info("update : SNAT disabled for Router {}", routerName);
1304                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1305                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1306                                     removeFlowInvTx);
1307                         } else {
1308                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1309                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1310                         }
1311                     }
1312                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1313                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1314                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1315                     }
1316
1317                     //Check if the Update is on External IPs
1318                     LOG.debug("update : Checking if this is update on External IPs");
1319                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1320                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1321
1322                     //Check if the External IPs are added during the update.
1323                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1324                     addedExternalIps.removeAll(originalExternalIps);
1325                     if (addedExternalIps.size() != 0) {
1326                         LOG.debug("update : Start processing of the External IPs addition during the update "
1327                                 + "operation");
1328                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1329                                 update.getExtGwMacAddress(), dpnId,
1330                                 update.getNetworkId());
1331
1332                         for (String addedExternalIp : addedExternalIps) {
1333                 /*
1334                     1) Do nothing in the IntExtIp model.
1335                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1336                  */
1337                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1338                             String externalIp = externalIpParts[0];
1339                             String externalIpPrefix = externalIpParts[1];
1340                             String externalpStr = externalIp + "/" + externalIpPrefix;
1341                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1342                                             + "router ID {} in the ExternalIpsCounter model.",
1343                                     externalpStr, routerId);
1344                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1345                         }
1346                         LOG.debug(
1347                                 "update : End processing of the External IPs addition during the update operation");
1348                     }
1349
1350                     //Check if the External IPs are removed during the update.
1351                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1352                     removedExternalIps.removeAll(updatedExternalIps);
1353                     if (removedExternalIps.size() > 0) {
1354                         LOG.debug("update : Start processing of the External IPs removal during the update "
1355                                 + "operation");
1356                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1357                                 removedExternalIps, original.getExtGwMacAddress(),
1358                                 dpnId, networkId);
1359
1360                         for (String removedExternalIp : removedExternalIps) {
1361                 /*
1362                     1) Remove the mappings in the IntExt IP model which has external IP.
1363                     2) Remove the external IP in the ExternalCounter model.
1364                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1365                        least loaded external IP.
1366                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1367                     4) Increase the count of the allocated external IP by one.
1368                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1369                      i.e. the route for the external IP is absent.
1370                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1371                      the removed external IPs and also from the model.
1372                     7) Advertise to the BGP for removing the route for the removed external IPs.
1373                  */
1374
1375                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1376                             String externalIp = externalIpParts[0];
1377                             String externalIpPrefix = externalIpParts[1];
1378                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1379
1380                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1381                                     + "entries for removed external IP {}", externalIpAddrStr);
1382                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1383                             String vpnName = "";
1384                             if (vpnUuId != null) {
1385                                 vpnName = vpnUuId.getValue();
1386                             }
1387                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1388                                     update.getExtGwMacAddress(), removeFlowInvTx);
1389
1390                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1391                             //Get the internal IPs which are associated to the removed external IPs
1392                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1393                             List<String> removedInternalIps = new ArrayList<>();
1394                             for (IpMap ipMap : ipMaps) {
1395                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1396                                     removedInternalIps.add(ipMap.getInternalIp());
1397                                 }
1398                             }
1399
1400                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1401                             for (String removedInternalIp : removedInternalIps) {
1402                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1403                                                 + "router ID {} from the IntExtIP model",
1404                                         removedInternalIp, routerId);
1405                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1406                             }
1407
1408                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1409                                             + "router ID {} from the ExternalIpsCounter model.",
1410                                     externalIpAddrStr, routerId);
1411                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1412
1413                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1414                                     + "whose external IPs were removed.");
1415                             for (String removedInternalIp : removedInternalIps) {
1416                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1417                                         removedInternalIp, writeFlowInvTx);
1418                             }
1419
1420                             LOG.debug("update : Remove the NAPT translation entries from "
1421                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1422                             //Get the internalIP and internal Port which were associated to the removed external IP.
1423                             List<Integer> externalPorts = new ArrayList<>();
1424                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1425                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1426                                     .builder(IntextIpPortMap.class)
1427                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1428                             Optional<IpPortMapping> ipPortMapping =
1429                                     MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1430                             if (ipPortMapping.isPresent()) {
1431                                 List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1432                                         .getIntextIpProtocolType();
1433                                 for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1434                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1435                                     List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1436                                     for (IpPortMap ipPortMap : ipPortMaps) {
1437                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1438                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1439                                             externalPorts.add(ipPortExternal.getPortNum());
1440                                             List<String> removedInternalIpPorts =
1441                                                     protoTypesIntIpPortsMap.get(protoType);
1442                                             if (removedInternalIpPorts != null) {
1443                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1444                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1445                                             } else {
1446                                                 removedInternalIpPorts = new ArrayList<>();
1447                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1448                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1449                                             }
1450                                         }
1451                                     }
1452                                 }
1453                             }
1454
1455                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1456                             // the removed external IP.
1457                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1458                                     protoTypesIntIpPortsMap.entrySet();
1459                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1460                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1461                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1462                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1463                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1464                                     // Remove the IP port map from the intext-ip-port-map model,
1465                                     // which were containing the removed external IP
1466                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1467                                             protocolType);
1468                                     //Remove the IP port incomint packer map.
1469                                     naptPacketInHandler.removeIncomingPacketMap(
1470                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1471                                     String[] removedInternalIpPortParts = removedInternalIpPort
1472                                             .split(NatConstants.COLON_SEPARATOR);
1473                                     if (removedInternalIpPortParts.length == 2) {
1474                                         String removedInternalIp = removedInternalIpPortParts[0];
1475                                         String removedInternalPort = removedInternalIpPortParts[1];
1476                                         List<String> removedInternalPortsList =
1477                                                 internalIpPortMap.get(removedInternalPort);
1478                                         if (removedInternalPortsList != null) {
1479                                             removedInternalPortsList.add(removedInternalPort);
1480                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1481                                         } else {
1482                                             removedInternalPortsList = new ArrayList<>();
1483                                             removedInternalPortsList.add(removedInternalPort);
1484                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1485                                         }
1486                                     }
1487                                 }
1488                             }
1489
1490                             // Delete the entry from SnatIntIpPortMap DS
1491                             Set<String> internalIps = internalIpPortMap.keySet();
1492                             for (String internalIp : internalIps) {
1493                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1494                                         + "model SnatIntIpPortMap", internalIp);
1495                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1496                             }
1497
1498                             naptManager.removeNaptPortPool(externalIp);
1499
1500                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1501                                     + "the removed external IP {}", externalIp);
1502                             for (Integer externalPort : externalPorts) {
1503                                 //Remove the NAPT translation entries from Inbound NAPT table
1504                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1505                                         routerId, externalIp, externalPort);
1506                             }
1507
1508                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1509                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1510                                 String internalIp = internalIpPort.getKey();
1511                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1512                                         + "for the removed internal IP {}", internalIp);
1513                                 List<String> internalPorts = internalIpPort.getValue();
1514                                 for (String internalPort : internalPorts) {
1515                                     //Remove the NAPT translation entries from Outbound NAPT table
1516                                     naptPacketInHandler.removeIncomingPacketMap(
1517                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1518                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1519                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1520                                             routerId, internalIp, Integer.parseInt(internalPort));
1521                                 }
1522                             }
1523                         }
1524                         LOG.debug(
1525                                 "update : End processing of the External IPs removal during the update operation");
1526                     }
1527
1528                     //Check if its Update on subnets
1529                     LOG.debug("update : Checking if this is update on subnets");
1530                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1531                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1532                     Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1533                     addedSubnetIds.removeAll(originalSubnetIds);
1534
1535                     //Check if the Subnet IDs are added during the update.
1536                     if (addedSubnetIds.size() != 0) {
1537                         LOG.debug(
1538                                 "update : Start processing of the Subnet IDs addition during the update operation");
1539                         for (Uuid addedSubnetId : addedSubnetIds) {
1540                 /*
1541                     1) Select the least loaded external IP for the subnet and store the mapping of the
1542                     subnet IP and the external IP in the IntExtIp model.
1543                     2) Increase the count of the selected external IP by one.
1544                     3) Advertise to the BGP if external IP is allocated for the first time for the
1545                     router i.e. the route for the external IP is absent.
1546                  */
1547                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1548                             if (subnetIp != null) {
1549                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1550                                         writeFlowInvTx);
1551                             }
1552                         }
1553                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1554                     }
1555
1556                     //Check if the Subnet IDs are removed during the update.
1557                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1558                     removedSubnetIds.removeAll(updatedSubnetIds);
1559                     if (removedSubnetIds.size() != 0) {
1560                         LOG.debug(
1561                                 "update : Start processing of the Subnet IDs removal during the update operation");
1562                         for (Uuid removedSubnetId : removedSubnetIds) {
1563                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1564                             if (subnetAddr != null) {
1565                     /*
1566                         1) Remove the subnet IP and the external IP in the IntExtIp map
1567                         2) Decrease the count of the coresponding external IP by one.
1568                         3) Advertise to the BGP for removing the routes of the corresponding external
1569                         IP if its not allocated to any other internal IP.
1570                     */
1571
1572                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1573                                         subnetAddr[0] + "/" + subnetAddr[1]);
1574                                 if (externalIp == null) {
1575                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1576                                             routerId, subnetAddr[0]);
1577                                     return;
1578                                 }
1579
1580                                 naptManager.updateCounter(routerId, externalIp, false);
1581                                 // Traverse entire model of external-ip counter whether external ip is not
1582                                 // used by any other internal ip in any router
1583                                 if (!isExternalIpAllocated(externalIp)) {
1584                                     LOG.debug("update : external ip is not allocated to any other "
1585                                             + "internal IP so proceeding to remove routes");
1586                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1587                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1588                                             removeFlowInvTx);
1589                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1590                                                     + "router {} with networkId {} and externalIp {}",
1591                                             dpnId, routerId, networkId, externalIp);
1592                                 }
1593
1594                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1595                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1596                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1597                             }
1598                         }
1599                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1600                     }
1601                 }));
1602             }));
1603             return futures;
1604         }, NatConstants.NAT_DJC_MAX_RETRIES);
1605     }
1606
1607     private boolean isExternalIpAllocated(String externalIp) {
1608         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1609         Optional<ExternalIpsCounter> externalCountersData =
1610             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1611         if (externalCountersData.isPresent()) {
1612             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1613             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1614             for (ExternalCounters ext : externalCounters) {
1615                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1616                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1617                         if (externalIpCount.getCounter() != 0) {
1618                             return true;
1619                         }
1620                         break;
1621                     }
1622                 }
1623             }
1624         }
1625         return false;
1626     }
1627
1628     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1629             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1630         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1631         try {
1632             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1633             if (address instanceof Inet6Address) {
1634                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1635                 return;
1636             }
1637         } catch (UnknownHostException e) {
1638             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1639             return;
1640         }
1641         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1642         if (leastLoadedExtIpAddr != null) {
1643             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1644             String leastLoadedExtIp = externalIpParts[0];
1645             String leastLoadedExtIpPrefix = externalIpParts[1];
1646             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1647             subnetIp = subnetIpParts[0];
1648             String subnetIpPrefix = subnetIpParts[1];
1649             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1650             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1651                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1652                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1653             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1654
1655
1656             // Check if external IP is already assigned a route. (i.e. External IP is previously
1657             // allocated to any of the subnets)
1658             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1659             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1660             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1661             if (label != null) {
1662                 //update
1663                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1664                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1665                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1666                     label, internalIp, leastLoadedExtIpAddrStr);
1667                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1668                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1669                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1670                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1671                 return;
1672             }
1673
1674             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1675             // for the first time and hence not having a route.
1676             //Get the VPN Name using the network ID
1677             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1678             if (vpnName != null) {
1679                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1680                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1681                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1682                         + "added after gateway-set");
1683                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1684                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1685                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1686                         return;
1687                     }
1688                 }
1689                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1690                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1691                     writeFlowInvTx);
1692             }
1693         }
1694     }
1695
1696     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1697         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1698         for (IpMap ipMap : ipMaps) {
1699             if (ipMap.getExternalIp().equals(externalIp)) {
1700                 if (ipMap.getLabel() != null) {
1701                     return ipMap.getLabel();
1702                 }
1703             }
1704         }
1705         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1706         return null;
1707     }
1708
1709     @Override
1710     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1711         LOG.trace("remove : Router delete method");
1712         /*
1713         ROUTER DELETE SCENARIO
1714         1) Get the router ID from the event.
1715         2) Build the cookie information from the router ID.
1716         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1717         4) Build the flow with the cookie value.
1718         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1719         6) Remove the flows from the other switches which points to the primary and secondary
1720          switches for the flows related the router ID.
1721         7) Get the list of external IP address maintained for the router ID.
1722         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1723         9) Withdraw the corresponding routes from the BGP.
1724          */
1725
1726         if (identifier == null || router == null) {
1727             LOG.error("remove : returning without processing since routers is null");
1728             return;
1729         }
1730
1731         String routerName = router.getRouterName();
1732         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1733             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1734                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1735                         routerName);
1736                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1737                 if (routerId == NatConstants.INVALID_ID) {
1738                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1739                             routerName);
1740                     return;
1741                 }
1742                 long bgpVpnId = NatConstants.INVALID_ID;
1743                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1744                 if (bgpVpnUuid != null) {
1745                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1746                 }
1747                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1748                         tx);
1749                 Uuid networkUuid = router.getNetworkId();
1750
1751                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1752                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1753                     // No NAPT switch for external router, probably because the router is not attached to
1754                     // any
1755                     // internal networks
1756                     LOG.debug(
1757                             "No NAPT switch for router {}, check if router is attached to any internal "
1758                                     + "network",
1759                             routerName);
1760                     return;
1761                 } else {
1762                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1763                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1764                             routerId, tx);
1765                 }
1766                 NatOverVxlanUtil.releaseVNI(routerName, idManager);
1767             })), NatConstants.NAT_DJC_MAX_RETRIES);
1768     }
1769
1770     // TODO Clean up the exception handling
1771     @SuppressWarnings("checkstyle:IllegalCatch")
1772     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1773                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1774                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1775         LOG.info("handleDisableSnat : Entry");
1776         String routerName = router.getRouterName();
1777         try {
1778             if (routerFlag) {
1779                 removeNaptSwitch(routerName);
1780             } else {
1781                 updateNaptSwitch(routerName, BigInteger.ZERO);
1782             }
1783
1784             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1785             naptManager.removeExternalCounter(routerId);
1786
1787             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1788             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1789                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1790                     + "router ID {} from RouterNaptSwitch model", routerId);
1791                 return;
1792             }
1793             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1794                     networkUuid);
1795             if (extNwProvType == null) {
1796                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1797                 return;
1798             }
1799             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1800             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1801                     externalSubnetList, removeFlowInvTx, extNwProvType);
1802             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1803             try {
1804                 String externalSubnetVpn = null;
1805                 for (Uuid externalSubnetId : externalSubnetList) {
1806                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1807                     // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1808                     if (externalSubnet.isPresent()) {
1809                         externalSubnetVpn =  externalSubnetId.getValue();
1810                         clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1811                                 router.getExtGwMacAddress(), removeFlowInvTx);
1812                     }
1813                 }
1814                 if (externalSubnetVpn == null) {
1815                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1816                             router.getExtGwMacAddress(), removeFlowInvTx);
1817                 }
1818             } catch (Exception ex) {
1819                 LOG.error("handleDisableSnat : Failed to remove fib entries for routerId {} in naptSwitchDpnId {}",
1820                     routerId, naptSwitchDpnId, ex);
1821             }
1822             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1823             // for the router ID.
1824             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1825                     + "router ID {} in the DS", routerId);
1826             naptManager.removeMapping(routerId);
1827         } catch (Exception ex) {
1828             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, ex);
1829         }
1830         LOG.info("handleDisableSnat : Exit");
1831     }
1832
1833     // TODO Clean up the exception handling
1834     @SuppressWarnings("checkstyle:IllegalCatch")
1835     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1836                                              @Nonnull Collection<String> externalIps,
1837                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1838         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1839                 + "with internet vpn {}", routerName, vpnId);
1840         try {
1841             BigInteger naptSwitchDpnId = null;
1842             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1843                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1844             Optional<RouterToNaptSwitch> rtrToNapt =
1845                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1846             if (rtrToNapt.isPresent()) {
1847                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1848             }
1849             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1850
1851             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1852                     writeFlowInvTx);
1853             try {
1854                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1855                 if (extGwMacAddress != null) {
1856                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1857                             + "External Router ID {}", extGwMacAddress, routerId);
1858                 } else {
1859                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1860                             + "External Router ID {}", routerId);
1861                     return;
1862                 }
1863                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1864                         writeFlowInvTx);
1865             } catch (Exception ex) {
1866                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1867                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1868             }
1869             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1870         } catch (Exception ex) {
1871             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1872                     + "with internet vpn {}", routerName, vpnId, ex);
1873         }
1874         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1875                 routerName, vpnId);
1876     }
1877
1878     // TODO Clean up the exception handling
1879     @SuppressWarnings("checkstyle:IllegalCatch")
1880     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1881         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1882             .setPrimarySwitchId(naptSwitchId).build();
1883         try {
1884             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1885                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1886         } catch (Exception ex) {
1887             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1888                 naptSwitchId, routerName);
1889         }
1890         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1891             naptSwitchId, routerName);
1892     }
1893
1894     protected void removeNaptSwitch(String routerName) {
1895         // Remove router and switch from model
1896         InstanceIdentifier<RouterToNaptSwitch> id =
1897             InstanceIdentifier.builder(NaptSwitches.class)
1898                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1899         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1900         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1901         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1902         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1903     }
1904
1905     // TODO skitt Fix the exception handling here
1906     @SuppressWarnings("checkstyle:IllegalCatch")
1907     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
1908     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1909                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1910                                                 @Nonnull Collection<String> externalIps,
1911                                                 Collection<Uuid> externalSubnetList,
1912                                                 TypedReadWriteTransaction<Configuration> confTx,
1913                                                 ProviderTypes extNwProvType) {
1914         try {
1915             LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1916             BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1917
1918             //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1919             // traffic which comes from the  VMs of the NAPT switches)
1920             String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1921             FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1922
1923             LOG.info(
1924                 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1925                     + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1926             mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1927
1928             //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1929             // traffic which comes from the VMs of the non NAPT switches)
1930             long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1931                 routerName);
1932             String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1933             FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1934             LOG.info(
1935                 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1936                     + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1937             mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1938
1939             //Remove the flow table 25->44 from NAPT Switch
1940             if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1941                 NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1942             }
1943
1944             //Remove the Outbound flow entry which forwards the packet to FIB Table
1945             LOG.info(
1946                 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1947                     + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1948
1949             String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1950                 NwConstants.IP_PROT_TCP);
1951             FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1952                 outboundTcpNatFlowRef);
1953             mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1954
1955             String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1956                 NwConstants.IP_PROT_UDP);
1957             FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1958                 outboundUdpNatFlowRef);
1959             mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1960
1961             String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1962                 NwConstants.IP_PROT_ICMP);
1963             FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1964                 icmpDropFlowRef);
1965             mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1966             boolean lastRouterOnExternalNetwork =
1967                 !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1968             if (lastRouterOnExternalNetwork) {
1969                 removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1970             }
1971             //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1972             // External Subnet Vpn Id.
1973             for (Uuid externalSubnetId : externalSubnetList) {
1974                 long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1975                 if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1976                     dataBroker, externalSubnetId, routerName, dpnId)) {
1977                     String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1978                     FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1979                         natPfibSubnetFlowRef);
1980                     mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1981                     LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1982                               + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1983                         subnetVpnId, dpnId);
1984                 }
1985             }
1986
1987             //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1988             String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1989             FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1990
1991             LOG.info(
1992                 "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1993                 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1994             mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1995
1996             if (lastRouterOnExternalNetwork) {
1997                 // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1998                 // - This does not work since ext-routers is deleted already - no network info
1999                 //Get the VPN ID from the ExternalNetworks model
2000                 long vpnId = -1;
2001                 if (vpnName == null || vpnName.isEmpty()) {
2002                     // ie called from router delete cases
2003                     Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2004                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2005                     if (vpnUuid != null) {
2006                         vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2007                         LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2008                                   + "or disableSNAT scenario", vpnId, networkId);
2009                     }
2010                 } else {
2011                     // ie called from disassociate vpn case
2012                     LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2013                         vpnName);
2014                     vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2015                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2016                         vpnId);
2017                 }
2018
2019                 if (vpnId != NatConstants.INVALID_ID) {
2020                     //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2021                     String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2022                     FlowEntity natPfibVpnFlowEntity =
2023                         NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2024                     LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2025                              + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2026                     mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2027                 }
2028             }
2029
2030             //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2031             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2032             if (ipPortMapping == null) {
2033                 LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2034                 return;
2035             }
2036
2037             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2038             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2039                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2040                 for (IpPortMap ipPortMap : ipPortMaps) {
2041                     String ipPortInternal = ipPortMap.getIpPortInternal();
2042                     String[] ipPortParts = ipPortInternal.split(":");
2043                     if (ipPortParts.length != 2) {
2044                         LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2045                         return;
2046                     }
2047                     String internalIp = ipPortParts[0];
2048                     String internalPort = ipPortParts[1];
2049
2050                     //Build the flow for the outbound NAPT table
2051                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2052                         + NatConstants.COLON_SEPARATOR + internalPort);
2053                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2054                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2055                     FlowEntity outboundNaptFlowEntity =
2056                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2057
2058                     LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2059                         + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2060                     mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2061
2062                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2063                     String externalIp = ipPortExternal.getIpAddress();
2064                     int externalPort = ipPortExternal.getPortNum();
2065
2066                     //Build the flow for the inbound NAPT table
2067                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2068                         String.valueOf(routerId), externalIp, externalPort);
2069                     FlowEntity inboundNaptFlowEntity =
2070                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2071
2072                     LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2073                         + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2074                     mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2075                 }
2076             }
2077         } catch (Exception e) {
2078             LOG.error("Error removing flow", e);
2079             throw new RuntimeException("Error removing flow", e);
2080         }
2081     }
2082
2083     // TODO skitt Fix the exception handling here
2084     @SuppressWarnings("checkstyle:IllegalCatch")
2085     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
2086     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2087                                                     @Nonnull Collection<String> externalIps,
2088                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2089         long extVpnId = NatConstants.INVALID_ID;
2090         if (networkId != null) {
2091             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2092             if (vpnUuid != null) {
2093                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2094             } else {
2095                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2096             }
2097         } else {
2098             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2099             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2100         }
2101         if (extVpnId == NatConstants.INVALID_ID) {
2102             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2103             extVpnId = routerId;
2104         }
2105         for (String ip : externalIps) {
2106             String extIp = removeMaskFromIp(ip);
2107             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2108             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2109                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2110                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2111             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2112             try {
2113                 mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2114             } catch (Exception e) {
2115                 LOG.error("Error removing flow", e);
2116                 throw new RuntimeException("Error removing flow", e);
2117             }
2118         }
2119     }
2120
2121     private String removeMaskFromIp(String ip) {
2122         if (ip != null && !ip.trim().isEmpty()) {
2123             return ip.split("/")[0];
2124         }
2125         return ip;
2126     }
2127
2128     // TODO skitt Fix the exception handling here
2129     @SuppressWarnings("checkstyle:IllegalCatch")
2130     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
2131     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2132                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2133                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2134         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2135         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2136
2137         //Remove the NAPT PFIB TABLE entry
2138         long vpnId = -1;
2139         if (vpnName != null) {
2140             // ie called from disassociate vpn case
2141             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2142                     + "with vpnName {}", vpnName);
2143             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2144             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2145                     vpnId);
2146         }
2147
2148         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2149                 networkId, routerName, dpnId)) {
2150             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2151             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2152             FlowEntity natPfibVpnFlowEntity =
2153                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2154             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2155                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2156             try {
2157                 mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2158             } catch (Exception e) {
2159                 LOG.error("Error removing flow", e);
2160                 throw new RuntimeException("Error removing flow", e);
2161             }
2162
2163             // Remove IP-PORT active NAPT entries and release port from IdManager
2164             // For the router ID get the internal IP , internal port and the corresponding
2165             // external IP and external Port.
2166             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2167             if (ipPortMapping == null) {
2168                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2169                 return;
2170             }
2171             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2172             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2173                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2174                 for (IpPortMap ipPortMap : ipPortMaps) {
2175                     String ipPortInternal = ipPortMap.getIpPortInternal();
2176                     String[] ipPortParts = ipPortInternal.split(":");
2177                     if (ipPortParts.length != 2) {
2178                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2179                                 + "and port");
2180                         return;
2181                     }
2182                     String internalIp = ipPortParts[0];
2183                     String internalPort = ipPortParts[1];
2184
2185                     //Build the flow for the outbound NAPT table
2186                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2187                             + NatConstants.COLON_SEPARATOR + internalPort);
2188                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2189                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2190                     FlowEntity outboundNaptFlowEntity =
2191                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2192
2193                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2194                             + "active switch with the DPN ID {} and router ID {}",
2195                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2196                     try {
2197                         mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2198                     } catch (Exception e) {
2199                         LOG.error("Error removing flow", e);
2200                         throw new RuntimeException("Error removing flow", e);
2201                     }
2202
2203                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2204                     String externalIp = ipPortExternal.getIpAddress();
2205                     int externalPort = ipPortExternal.getPortNum();
2206
2207                     //Build the flow for the inbound NAPT table
2208                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2209                         String.valueOf(routerId), externalIp, externalPort);
2210                     FlowEntity inboundNaptFlowEntity =
2211                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2212
2213                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2214                             + "active active switch with the DPN ID {} and router ID {}",
2215                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2216                     try {
2217                         mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2218                     } catch (Exception e) {
2219                         LOG.error("Error removing flow", e);
2220                         throw new RuntimeException("Error removing flow", e);
2221                     }
2222
2223                     // Finally release port from idmanager
2224                     String internalIpPort = internalIp + ":" + internalPort;
2225                     naptManager.removePortFromPool(internalIpPort, externalIp);
2226
2227                     //Remove sessions from models
2228                     naptManager.removeIpPortMappingForRouterID(routerId);
2229                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2230                 }
2231             }
2232         } else {
2233             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2234         }
2235     }
2236
2237     // TODO skitt Fix the exception handling here
2238     @SuppressWarnings("checkstyle:IllegalCatch")
2239     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
2240     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2241             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2242         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2243
2244         // Remove the flows from the other switches which points to the primary and secondary switches
2245         // for the flows related the router ID.
2246         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2247         if (allSwitchList.isEmpty()) {
2248             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2249             return;
2250         }
2251         try {
2252             for (BigInteger dpnId : allSwitchList) {
2253                 if (!naptSwitchDpnId.equals(dpnId)) {
2254                     LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2255
2256                     //Remove the PSNAT entry which forwards the packet to Terminating Service table
2257                     String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2258                     FlowEntity preSnatFlowEntity =
2259                         NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2260
2261                     LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2262                         + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2263                     mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2264
2265                     //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2266                     long groupId = createGroupId(getGroupIdKey(routerName));
2267                     List<BucketInfo> listBucketInfo = new ArrayList<>();
2268                     GroupEntity preSnatGroupEntity =
2269                         MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, listBucketInfo);
2270
2271                     LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2272                         + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2273                     mdsalManager.removeGroup(removeFlowInvTx, preSnatGroupEntity);
2274
2275                 }
2276             }
2277         } catch (Exception e) {
2278             LOG.error("Error removing flow", e);
2279             throw new RuntimeException("Error removing flow", e);
2280         }
2281     }
2282
2283     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2284                                          @Nonnull Collection<String> externalIps, String vpnName,
2285                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx) {
2286         //Withdraw the corresponding routes from the BGP.
2287         //Get the network ID using the router ID.
2288         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2289                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2290         if (networkUuid == null) {
2291             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2292             return;
2293         }
2294
2295         if (externalIps.isEmpty()) {
2296             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2297             return;
2298         }
2299
2300         if (vpnName == null) {
2301             //Get the VPN Name using the network ID
2302             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2303             if (vpnName == null) {
2304                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2305                     networkUuid, routerId);
2306                 return;
2307             }
2308         }
2309         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2310
2311         //Remove custom FIB routes
2312         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2313         for (String extIp : externalIps) {
2314             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2315         }
2316     }
2317
2318     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2319                                             final Uuid networkUuid, String extGwMacAddress,
2320                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2321         clearBgpRoutes(extIp, vpnName);
2322         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2323                 removeFlowInvTx);
2324     }
2325
2326     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2327                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2328                                              String gwMacAddress, boolean switchOver,
2329                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2330         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2331         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2332         if (routerName == null) {
2333             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2334             return;
2335         }
2336         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2337         if (extNwProvType == null) {
2338             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2339             return;
2340         }
2341         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2342          * external network provided type is VxLAN
2343          */
2344         if (extNwProvType == ProviderTypes.VXLAN) {
2345             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2346             );
2347             return;
2348         }
2349         if (tempLabel < 0) {
2350             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2351                     extIp, routerId);
2352             return;
2353         }
2354
2355         final long label = tempLabel;
2356         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2357         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2358                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2359                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2360         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2361
2362         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2363         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2364         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2365             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2366             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2367         }
2368         if (!switchOver) {
2369             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2370                 Futures.transformAsync(future, result -> {
2371                     //Release label
2372                     if (result.isSuccessful()) {
2373                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2374                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2375                             .setVpnName(vpnName).setIpPrefix(externalIp).build();
2376                         return vpnService.removeVpnLabel(labelInput);
2377                     } else {
2378                         String errMsg =
2379                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2380                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2381                         LOG.error(errMsg);
2382                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2383                     }
2384                 }, MoreExecutors.directExecutor());
2385
2386             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2387
2388                 @Override
2389                 public void onFailure(@Nonnull Throwable error) {
2390                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2391                         + "got external ip {}", label, extIp, error);
2392                 }
2393
2394                 @Override
2395                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2396                     if (result.isSuccessful()) {
2397                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2398                             + "from VPN {}", externalIp, vpnName);
2399                     } else {
2400                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2401                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2402                     }
2403                 }
2404             }, MoreExecutors.directExecutor());
2405         } else {
2406             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2407                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2408         }
2409     }
2410
2411     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2412                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2413                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2414         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2415         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2416         if (routerName == null) {
2417             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2418             return;
2419         }
2420         //Get the external network provider type from networkId
2421         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2422         if (extNwProvType == null) {
2423             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2424                     networkUuid);
2425             return;
2426         }
2427         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2428          *  external network provided type is VxLAN
2429          */
2430         if (extNwProvType == ProviderTypes.VXLAN) {
2431             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2432             return;
2433         }
2434         //Get IPMaps from the DB for the router ID
2435         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2436         if (dbIpMaps.isEmpty()) {
2437             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2438             return;
2439         }
2440
2441         long tempLabel = NatConstants.INVALID_ID;
2442         for (IpMap dbIpMap : dbIpMaps) {
2443             String dbExternalIp = dbIpMap.getExternalIp();
2444             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2445             //Select the IPMap, whose external IP is the IP for which FIB is installed
2446             if (extIp.equals(dbExternalIp)) {
2447                 tempLabel = dbIpMap.getLabel();
2448                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2449                     tempLabel, dbExternalIp, routerId);
2450                 break;
2451             }
2452         }
2453         if (tempLabel == NatConstants.INVALID_ID) {
2454             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2455                     extIp, routerId);
2456             return;
2457         }
2458
2459         final long label = tempLabel;
2460         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2461         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2462                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2463                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2464         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2465
2466         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2467         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2468         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2469             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2470             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2471         }
2472         if (!switchOver) {
2473             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2474                     Futures.transformAsync(future, result -> {
2475                         //Release label
2476                         if (result.isSuccessful()) {
2477                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2478                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2479                             return vpnService.removeVpnLabel(labelInput);
2480                         } else {
2481                             String errMsg =
2482                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2483                                             + "prefix %s Failed - %s",
2484                                             dpnId, externalIp, result.getErrors());
2485                             LOG.error(errMsg);
2486                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2487                         }
2488                     }, MoreExecutors.directExecutor());
2489
2490             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2491
2492                 @Override
2493                 public void onFailure(@Nonnull Throwable error) {
2494                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2495                 }
2496
2497                 @Override
2498                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2499                     if (result.isSuccessful()) {
2500                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2501                             + "from VPN {}", externalIp, vpnName);
2502                     } else {
2503                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2504                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2505                     }
2506                 }
2507             }, MoreExecutors.directExecutor());
2508         } else {
2509             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2510                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2511         }
2512     }
2513
2514     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2515                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2516                                                TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2517         //Withdraw the corresponding routes from the BGP.
2518         //Get the network ID using the router ID.
2519         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2520                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2521         if (networkUuid == null) {
2522             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2523             return;
2524         }
2525
2526         if (externalIps == null || externalIps.isEmpty()) {
2527             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2528             return;
2529         }
2530
2531         if (vpnName == null) {
2532             //Get the VPN Name using the network ID
2533             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2534             if (vpnName == null) {
2535                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2536                     networkUuid, routerId);
2537                 return;
2538             }
2539         }
2540         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2541
2542         //Remove custom FIB routes
2543         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2544         for (String extIp : externalIps) {
2545             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2546                     writeFlowInvTx);
2547         }
2548     }
2549
2550     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2551         //Inform BGP about the route removal
2552         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2553         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2554         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2555     }
2556
2557     // TODO skitt Fix the exception handling here
2558     @SuppressWarnings("checkstyle:IllegalCatch")
2559     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
2560     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2561         TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2562         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2563         List<MatchInfo> mkMatches = new ArrayList<>();
2564         // Matching metadata
2565         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
2566         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
2567             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
2568             5, String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
2569             COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
2570         try {
2571             mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowEntity);
2572         } catch (Exception e) {
2573             LOG.error("Error removing flow", e);
2574             throw new RuntimeException("Error removing flow", e);
2575         }
2576         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2577     }
2578
2579     // TODO skitt Fix the exception handling here
2580     @SuppressWarnings("checkstyle:IllegalCatch")
2581     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
2582     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2583         TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2584         List<MatchInfo> matches = new ArrayList<>();
2585         matches.add(MatchEthernetType.MPLS_UNICAST);
2586         matches.add(new MatchMplsLabel(serviceId));
2587
2588         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2589
2590         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2591
2592         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
2593             10, flowRef, 0, 0,
2594             COOKIE_VM_LFIB_TABLE, matches, null);
2595
2596         try {
2597             mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowEntity);
2598         } catch (Exception e) {
2599             LOG.error("Error removing flow", e);
2600             throw new RuntimeException("Error removing flow", e);
2601         }
2602
2603         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2604     }
2605
2606     /**
2607      * router association to vpn.
2608      *
2609      * @param routerName - Name of router
2610      * @param routerId - router id
2611      * @param bgpVpnName BGP VPN name
2612      */
2613     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2614             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2615         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2616         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2617             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2618
2619             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2620
2621             if (bgpVpnId != NatConstants.INVALID_ID) {
2622                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2623                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2624                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2625                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2626                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2627                     getRoutersIdentifier(bgpVpnId), rtrs);
2628
2629                 // Get the allocated Primary NAPT Switch for this router
2630                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2631
2632                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2633                         routerId, bgpVpnId);
2634                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2635
2636                 // Get the group ID
2637                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2638                 createGroupId(getGroupIdKey(routerName));
2639                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2640                         extNwProvType);
2641             }
2642         }
2643     }
2644
2645     /**
2646      * router disassociation from vpn.
2647      *
2648      * @param routerName - Name of router
2649      * @param routerId - router id
2650      * @param bgpVpnName BGP VPN name
2651      */
2652     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2653             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2654         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2655         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2656             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2657             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2658
2659             // Get the allocated Primary NAPT Switch for this router
2660             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2661
2662             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2663             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2664
2665             // Get the group ID
2666             createGroupId(getGroupIdKey(routerName));
2667             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2668             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2669                     writeFlowInvTx, extNwProvType);
2670         }
2671     }
2672
2673     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2674         InstanceIdentifier<Routers> routerInstanceIndentifier =
2675             InstanceIdentifier.builder(ExtRouters.class)
2676                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2677         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2678         return routerData.isPresent() && routerData.get().isEnableSnat();
2679     }
2680
2681     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2682         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2683
2684         long changedVpnId = bgpVpnId;
2685         String idType = "BGP VPN";
2686         if (bgpVpnId == NatConstants.INVALID_ID) {
2687             changedVpnId = routerId;
2688             idType = "router";
2689         }
2690
2691         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2692         if (switches.isEmpty()) {
2693             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2694             return;
2695         }
2696         for (BigInteger dpnId : switches) {
2697             // Update the BGP VPN ID in the SNAT miss entry to group
2698             if (!dpnId.equals(primarySwitchId)) {
2699                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2700                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2701                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2702                 long groupId = createGroupId(getGroupIdKey(routerName));
2703                 if (!isSnatCfgd) {
2704                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2705                 }
2706
2707                 LOG.debug(
2708                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2709                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2710                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2711                 mdsalManager.addFlow(confTx, flowEntity);
2712             } else {
2713                 LOG.debug(
2714                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2715                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2716                 FlowEntity flowEntity =
2717                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2718                 mdsalManager.addFlow(confTx, flowEntity);
2719
2720                 LOG.debug(
2721                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2722                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2723                         idType, changedVpnId, primarySwitchId);
2724                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2725                         changedVpnId, confTx, extNwProvType);
2726
2727                 LOG.debug(
2728                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2729                                 + "which punts the packet to the controller in the Primary switch {}",
2730                         idType, changedVpnId, primarySwitchId);
2731                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2732
2733                 LOG.debug(
2734                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2735                                 + " outgoing packet to FIB Table in the Primary switch {}",
2736                         idType, changedVpnId, primarySwitchId);
2737                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2738
2739                 LOG.debug(
2740                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2741                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2742                                 + " {}", idType, changedVpnId, primarySwitchId);
2743                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2744
2745                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2746                         primarySwitchId);
2747                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2748                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2749                 if (vpnId != NatConstants.INVALID_ID) {
2750                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2751                 }
2752             }
2753         }
2754     }
2755
2756     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2757         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2758         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2759         if (ipPortMapping == null) {
2760             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2761             return;
2762         }
2763         // Get the External Gateway MAC Address
2764         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2765         if (extGwMacAddress != null) {
2766             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2767                     extGwMacAddress, routerId);
2768         } else {
2769             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2770                     routerId);
2771             return;
2772         }
2773         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2774         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2775             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2776             for (IpPortMap ipPortMap : ipPortMaps) {
2777                 String ipPortInternal = ipPortMap.getIpPortInternal();
2778                 String[] ipPortParts = ipPortInternal.split(":");
2779                 if (ipPortParts.length != 2) {
2780                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2781                     return;
2782                 }
2783                 String internalIp = ipPortParts[0];
2784                 String internalPort = ipPortParts[1];
2785                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2786                         internalIp, internalPort);
2787                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2788                 NAPTEntryEvent.Protocol protocol;
2789                 switch (protocolTypes) {
2790                     case TCP:
2791                         protocol = NAPTEntryEvent.Protocol.TCP;
2792                         break;
2793                     case UDP:
2794                         protocol = NAPTEntryEvent.Protocol.UDP;
2795                         break;
2796                     default:
2797                         protocol = NAPTEntryEvent.Protocol.TCP;
2798                 }
2799                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2800                 SessionAddress externalAddress =
2801                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2802                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2803                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2804                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2805                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2806                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2807             }
2808         }
2809     }
2810
2811     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2812                                                           long changedVpnId) {
2813
2814         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2815             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2816         List<MatchInfo> matches = new ArrayList<>();
2817         matches.add(MatchEthernetType.IPV4);
2818         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2819
2820         List<ActionInfo> actionsInfo = new ArrayList<>();
2821         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2822                 routerName);
2823         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2824         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2825                 actionsInfo);
2826         actionsInfo.add(new ActionGroup(groupId));
2827         List<InstructionInfo> instructions = new ArrayList<>();
2828         instructions.add(new InstructionApplyActions(actionsInfo));
2829         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2830         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2831             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2832             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2833
2834         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2835         return flowEntity;
2836     }
2837
2838     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2839                                                                         long changedVpnId) {
2840
2841         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2842                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2843         List<MatchInfo> matches = new ArrayList<>();
2844         matches.add(MatchEthernetType.IPV4);
2845         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2846
2847         List<InstructionInfo> instructions = new ArrayList<>();
2848         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2849
2850         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2851         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2852             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2853             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2854
2855         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2856         return flowEntity;
2857     }
2858
2859     // TODO : Replace this with ITM Rpc once its available with full functionality
2860     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2861         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2862
2863         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2864             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2865         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2866                 extNwProvType);
2867         mdsalManager.addFlow(confTx, flowEntity);
2868     }
2869
2870     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2871             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2872         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2873             dpId, routerName, changedVpnId);
2874         List<MatchInfo> matches = new ArrayList<>();
2875         matches.add(MatchEthernetType.IPV4);
2876
2877         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2878         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2879             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2880         }
2881         matches.add(new MatchTunnelId(tunnelId));
2882
2883         List<InstructionInfo> instructions = new ArrayList<>();
2884         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2885             MetaDataUtil.METADATA_MASK_VRFID));
2886         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2887         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2888         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2889         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2890             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2891             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2892         return flowEntity;
2893     }
2894
2895     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2896                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2897         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2898             dpnId, routerId, changedVpnId);
2899         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2900                 NwConstants.IP_PROT_TCP);
2901         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2902         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2903
2904         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2905                 NwConstants.IP_PROT_UDP);
2906         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2907         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2908
2909         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2910         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2911         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2912     }
2913
2914     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2915                                                            long changedVpnId, int protocol) {
2916         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2917             dpId, routerId, changedVpnId);
2918         BigInteger cookie = getCookieOutboundFlow(routerId);
2919         List<MatchInfo> matches = new ArrayList<>();
2920         matches.add(MatchEthernetType.IPV4);
2921         matches.add(new MatchIpProtocol((short)protocol));
2922         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2923
2924         List<InstructionInfo> instructions = new ArrayList<>();
2925         List<ActionInfo> actionsInfos = new ArrayList<>();
2926         actionsInfos.add(new ActionPuntToController());
2927         if (snatPuntTimeout != 0) {
2928             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2929         }
2930         instructions.add(new InstructionApplyActions(actionsInfos));
2931
2932         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2933         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2934             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2935         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2936         return flowEntity;
2937     }
2938
2939     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2940                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2941         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2942             dpnId, segmentId, changedVpnId);
2943         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2944         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2945     }
2946
2947     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2948
2949         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2950             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2951         List<MatchInfo> matches = new ArrayList<>();
2952         matches.add(MatchEthernetType.IPV4);
2953         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2954
2955         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2956         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2957         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2958         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2959         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2960
2961         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2962         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2963             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2964             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2965         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2966         return flowEntity;
2967     }
2968
2969     @Override
2970     protected ExternalRoutersListener getDataTreeChangeListener() {
2971         return ExternalRoutersListener.this;
2972     }
2973
2974     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2975                                                             TypedWriteTransaction<Configuration> writeFlowInvTx) {
2976         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2977                 routerName);
2978         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2979             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2980             if (subnetVpnId != -1) {
2981                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2982                     + "and vpnId {}", dpnId, subnetVpnId);
2983                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2984             }
2985         }
2986     }
2987 }