natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 148 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 149 import org.opendaylight.yangtools.yang.common.RpcResult;
 150 import org.slf4j.Logger;
 151 import org.slf4j.LoggerFactory;
 152
 153
 154 @Singleton
 155 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 156     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 157
 158     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 159     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 160
 161     private final DataBroker dataBroker;
 162     private final ManagedNewTransactionRunner txRunner;
 163     private final IMdsalApiManager mdsalManager;
 164     private final ItmRpcService itmManager;
 165     private final OdlInterfaceRpcService odlInterfaceRpcService;
 166     private final IdManagerService idManager;
 167     private final NaptManager naptManager;
 168     private final NAPTSwitchSelector naptSwitchSelector;
 169     private final IBgpManager bgpManager;
 170     private final VpnRpcService vpnService;
 171     private final FibRpcService fibService;
 172     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 173     private final NaptEventHandler naptEventHandler;
 174     private final NaptPacketInHandler naptPacketInHandler;
 175     private final IFibManager fibManager;
 176     private final IVpnManager vpnManager;
 177     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 178     private final NatMode natMode;
 179     private final IElanService elanManager;
 180     private final JobCoordinator coordinator;
 181     private final IInterfaceManager interfaceManager;
 182     private final NatOverVxlanUtil natOverVxlanUtil;
 183     private final int snatPuntTimeout;
 184
 185     @Inject
 186     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 187                                    final ItmRpcService itmManager,
 188                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 189                                    final IdManagerService idManager,
 190                                    final NaptManager naptManager,
 191                                    final NAPTSwitchSelector naptSwitchSelector,
 192                                    final IBgpManager bgpManager,
 193                                    final VpnRpcService vpnService,
 194                                    final FibRpcService fibService,
 195                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 196                                    final NaptEventHandler naptEventHandler,
 197                                    final NaptPacketInHandler naptPacketInHandler,
 198                                    final IFibManager fibManager,
 199                                    final IVpnManager vpnManager,
 200                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 201                                    final NatserviceConfig config,
 202                                    final IElanService elanManager,
 203                                    final JobCoordinator coordinator,
 204                                    final NatOverVxlanUtil natOverVxlanUtil,
 205                                    final IInterfaceManager interfaceManager) {
 206         super(Routers.class, ExternalRoutersListener.class);
 207         this.dataBroker = dataBroker;
 208         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 209         this.mdsalManager = mdsalManager;
 210         this.itmManager = itmManager;
 211         this.odlInterfaceRpcService = odlInterfaceRpcService;
 212         this.idManager = idManager;
 213         this.naptManager = naptManager;
 214         this.naptSwitchSelector = naptSwitchSelector;
 215         this.bgpManager = bgpManager;
 216         this.vpnService = vpnService;
 217         this.fibService = fibService;
 218         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 219         this.naptEventHandler = naptEventHandler;
 220         this.naptPacketInHandler = naptPacketInHandler;
 221         this.fibManager = fibManager;
 222         this.vpnManager = vpnManager;
 223         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 224         this.elanManager = elanManager;
 225         this.coordinator = coordinator;
 226         this.interfaceManager = interfaceManager;
 227         this.natOverVxlanUtil = natOverVxlanUtil;
 228         if (config != null) {
 229             this.natMode = config.getNatMode();
 230             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 231         } else {
 232             this.natMode = NatMode.Controller;
 233             this.snatPuntTimeout = 0;
 234         }
 235     }
 236
 237     @Override
 238     @PostConstruct
 239     public void init() {
 240         LOG.info("{} init", getClass().getSimpleName());
 241         // This class handles ExternalRouters for Controller SNAT mode.
 242         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 243         if (natMode == NatMode.Controller) {
 244             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 245             NatUtil.createGroupIdPool(idManager);
 246         }
 247     }
 248
 249     @Override
 250     protected InstanceIdentifier<Routers> getWildCardPath() {
 251         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 252     }
 253
 254     @Override
 255     // TODO Clean up the exception handling
 256     @SuppressWarnings("checkstyle:IllegalCatch")
 257     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 258         // Populate the router-id-name container
 259         String routerName = routers.getRouterName();
 260         LOG.info("add : external router event for {}", routerName);
 261         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 262         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 263         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 264         try {
 265             if (routers.isEnableSnat()) {
 266                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 267                     () -> Collections.singletonList(
 268                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 269                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 270                             long bgpVpnId = NatConstants.INVALID_ID;
 271                             if (bgpVpnUuid != null) {
 272                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 273                             }
 274                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 275                             // Allocate Primary Napt Switch for this router
 276                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 277                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 278                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 279                             }
 280                         }
 281                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 282             } else {
 283                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 284             }
 285         } catch (Exception ex) {
 286             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 287                     routerName, ex);
 288         }
 289     }
 290
 291     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 292                                  TypedWriteTransaction<Configuration> confTx) {
 293         String routerName = routers.getRouterName();
 294         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 295
 296         naptManager.initialiseExternalCounter(routers, routerId);
 297         subnetRegisterMapping(routers, routerId);
 298
 299         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 300             primarySwitchId, routerName);
 301
 302         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 303                 routers.getNetworkId());
 304         if (extNwProvType == null) {
 305             LOG.error("handleEnableSnat : External Network Provider Type missing");
 306             return;
 307         }
 308
 309         if (bgpVpnId != NatConstants.INVALID_ID) {
 310             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 311                     extNwProvType);
 312         } else {
 313             // write metadata and punt
 314             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 315             // Now install entries in SNAT tables to point to Primary for each router
 316             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 317             for (BigInteger dpnId : switches) {
 318                 // Handle switches and NAPT switches separately
 319                 if (!dpnId.equals(primarySwitchId)) {
 320                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 321                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 322                 } else {
 323                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 324                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 325                 }
 326             }
 327         }
 328
 329         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 330         if (externalIps.isEmpty()) {
 331             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 332             return;
 333         } else {
 334             for (String externalIpAddrPrefix : externalIps) {
 335                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 336                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 337                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 338                 );
 339             }
 340         }
 341         LOG.debug("handleEnableSnat : Exit");
 342     }
 343
 344     private BigInteger getPrimaryNaptSwitch(String routerName) {
 345         // Allocate Primary Napt Switch for this router
 346         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 347         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 348             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 349                 primarySwitchId, routerName);
 350             return primarySwitchId;
 351         }
 352         // Allocated an id from VNI pool for the Router.
 353         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 354         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 355         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 356
 357         return primarySwitchId;
 358     }
 359
 360     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 361                                                      TypedWriteTransaction<Configuration> confTx) {
 362         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 363         if (extVpnId == NatConstants.INVALID_ID) {
 364             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 365             extVpnId = routerId;
 366         }
 367         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 368         if (externalIps.isEmpty()) {
 369             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 370                 dpnId, extVpnId);
 371             return;
 372         }
 373         for (String ip : externalIps) {
 374             Uuid subnetId = getSubnetIdForFixedIp(ip);
 375             if (subnetId != null) {
 376                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 377                 if (subnetVpnId != NatConstants.INVALID_ID) {
 378                     extVpnId = subnetVpnId;
 379                 }
 380                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 381                     dpnId, extVpnId, subnetId);
 382                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 383                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 384             }
 385         }
 386     }
 387
 388     @Nullable
 389     private Uuid getSubnetIdForFixedIp(String ip) {
 390         if (ip != null) {
 391             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 392             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 393             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 394         }
 395         LOG.error("getSubnetIdForFixedIp : ip is null");
 396         return null;
 397     }
 398
 399     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 400         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 401         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 402         int counter = 0;
 403         int extIpCounter = externalIps.size();
 404         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 405                 counter, extIpCounter);
 406         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 407         if (subnetIds == null) {
 408             return;
 409         }
 410         for (Uuid subnet : subnetIds) {
 411             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 412             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 413                 .builder(Subnetmaps.class)
 414                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 415                 .build();
 416             Optional<Subnetmap> sn;
 417             try {
 418                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 419                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 420             } catch (ReadFailedException e) {
 421                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 422                 sn = Optional.absent();
 423             }
 424             if (sn.isPresent()) {
 425                 // subnets
 426                 Subnetmap subnetmapEntry = sn.get();
 427                 String subnetString = subnetmapEntry.getSubnetIp();
 428                 String[] subnetSplit = subnetString.split("/");
 429                 String subnetIp = subnetSplit[0];
 430                 try {
 431                     InetAddress address = InetAddress.getByName(subnetIp);
 432                     if (address instanceof Inet6Address) {
 433                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 434                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 435                         continue;
 436                     }
 437                 } catch (UnknownHostException e) {
 438                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 439                     return;
 440                 }
 441                 String subnetPrefix = "0";
 442                 if (subnetSplit.length == 2) {
 443                     subnetPrefix = subnetSplit[1];
 444                 }
 445                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 446                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 447                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 448                 //externalIps
 449                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 450                         counter, extIpCounter);
 451                 if (extIpCounter != 0) {
 452                     if (counter < extIpCounter) {
 453                         String[] ipSplit = externalIps.get(counter).split("/");
 454                         String externalIp = ipSplit[0];
 455                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 456                         if (ipSplit.length == 2) {
 457                             extPrefix = ipSplit[1];
 458                         }
 459                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 460                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 461                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 462                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 463                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 464                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 465                     } else {
 466                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 467                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 468                         String[] ipSplit = externalIps.get(counter).split("/");
 469                         String externalIp = ipSplit[0];
 470                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 471                         if (ipSplit.length == 2) {
 472                             extPrefix = ipSplit[1];
 473                         }
 474                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 475                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 476                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 477                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 478                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 479                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 480                             externalIp, extPrefix);
 481                     }
 482                 }
 483                 counter++;
 484                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 485             } else {
 486                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 487             }
 488         }
 489     }
 490
 491     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 492             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 493             throws ExecutionException, InterruptedException {
 494         //Check if BGP VPN exists. If exists then invoke the new method.
 495         if (bgpVpnId != NatConstants.INVALID_ID) {
 496             if (bgpVpnUuid != null) {
 497                 String bgpVpnName = bgpVpnUuid.getValue();
 498                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 499                     bgpVpnId, bgpVpnName);
 500                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 501                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 502                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 503             }
 504             if (create) {
 505                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 506             } else {
 507                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 508             }
 509             return;
 510         }
 511
 512         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 513         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 514     }
 515
 516     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 517             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 518         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 519         if (switches.isEmpty()) {
 520             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 521             return;
 522         }
 523         if (routerId == NatConstants.INVALID_ID) {
 524             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 525                     + "default NAT route in FIB", routerName);
 526             return;
 527         }
 528         for (BigInteger dpnId : switches) {
 529             if (create) {
 530                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 531                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 532                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 533             } else {
 534                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 535                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 536                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 537             }
 538         }
 539     }
 540
 541     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 542         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 543         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 544         if (dpnIds.isEmpty()) {
 545             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 546                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 547             return;
 548         }
 549         for (BigInteger dpnId : dpnIds) {
 550             if (bgpVpnId != NatConstants.INVALID_ID) {
 551                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 552                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 553                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 554             } else {
 555                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 556                     + "in dpn {} for the internal vpn", routerId, dpnId);
 557                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 558             }
 559         }
 560     }
 561
 562     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 563             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 564             throws ExecutionException, InterruptedException {
 565         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 566         if (dpnIds.isEmpty()) {
 567             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 568                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 569             return;
 570         }
 571         for (BigInteger dpnId : dpnIds) {
 572             if (bgpVpnId != NatConstants.INVALID_ID) {
 573                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 574                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 575                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 576             } else {
 577                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 578                     + "in dpn {} for the internal vpn", routerId, dpnId);
 579                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 580             }
 581         }
 582     }
 583
 584     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 585         TypedWriteTransaction<Configuration> confTx) {
 586         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 587         if (routerId != NatConstants.INVALID_ID) {
 588             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 589                     primarySwitchId, routerId);
 590             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 591         } else {
 592             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 593                 + "createAndInstallMissEntry", routerName);
 594         }
 595     }
 596
 597     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 598         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 599                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 600     }
 601
 602     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 603         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 604                 .FLOWID_SEPARATOR + vpnId;
 605     }
 606
 607     public BigInteger getCookieOutboundFlow(long routerId) {
 608         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 609             BigInteger.valueOf(routerId));
 610     }
 611
 612     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 613         long l4SrcPortField;
 614         long l4DstPortField;
 615         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 616
 617         if (protocol == NwConstants.IP_PROT_TCP) {
 618             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 619             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 620         } else {
 621             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 622             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 623         }
 624         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 625                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 626                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 627                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 628                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 629                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 630                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 631                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 632                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 633                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 634                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 635                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 636                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 637                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 638                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 639
 640         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 641                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 642     }
 643
 644     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 645         List<MatchInfo> matches = new ArrayList<>();
 646         matches.add(MatchEthernetType.IPV4);
 647         matches.add(MatchIpProtocol.ICMP);
 648         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 649
 650         List<ActionInfo> actionInfos = new ArrayList<>();
 651         actionInfos.add(new ActionDrop());
 652
 653         List<InstructionInfo> instructions = new ArrayList<>();
 654         instructions.add(new InstructionApplyActions(actionInfos));
 655
 656         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 657                 NwConstants.IP_PROT_ICMP);
 658         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 659                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 660                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 661         return flow;
 662     }
 663
 664     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 665         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 666         BigInteger cookie = getCookieOutboundFlow(routerId);
 667         List<MatchInfo> matches = new ArrayList<>();
 668         matches.add(MatchEthernetType.IPV4);
 669         matches.add(new MatchIpProtocol((short)protocol));
 670         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 671
 672         List<InstructionInfo> instructions = new ArrayList<>();
 673         List<ActionInfo> actionsInfos = new ArrayList<>();
 674         actionsInfos.add(new ActionPuntToController());
 675         if (snatPuntTimeout != 0) {
 676             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 677         }
 678         instructions.add(new InstructionApplyActions(actionsInfos));
 679
 680         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 681         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 682             5, flowRef, 0, 0,
 683             cookie, matches, instructions);
 684         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 685         return flowEntity;
 686     }
 687
 688     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 689         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 690         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 691         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 692         mdsalManager.addFlow(confTx, tcpflowEntity);
 693
 694         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 695         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 696         mdsalManager.addFlow(confTx, udpflowEntity);
 697
 698         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 699         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 700         mdsalManager.addFlow(confTx, icmpDropFlow);
 701     }
 702
 703     @Nullable
 704     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 705         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 706         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 707         try {
 708             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 709                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 710                     .setSourceDpid(srcDpId)
 711                     .setDestinationDpid(dstDpId)
 712                     .setTunnelType(tunType)
 713                     .build());
 714             rpcResult = result.get();
 715             if (!rpcResult.isSuccessful()) {
 716                 tunType = TunnelTypeGre.class;
 717                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 718                     .setSourceDpid(srcDpId)
 719                     .setDestinationDpid(dstDpId)
 720                     .setTunnelType(tunType)
 721                     .build());
 722                 rpcResult = result.get();
 723                 if (!rpcResult.isSuccessful()) {
 724                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 725                             rpcResult.getErrors());
 726                 } else {
 727                     return rpcResult.getResult().getInterfaceName();
 728                 }
 729                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 730                         rpcResult.getErrors());
 731             } else {
 732                 return rpcResult.getResult().getInterfaceName();
 733             }
 734         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 735             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 736                     + "between {} and {}", srcDpId, dstDpId, e);
 737         }
 738
 739         return null;
 740     }
 741
 742     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 743         TypedWriteTransaction<Configuration> confTx) {
 744
 745         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 746         // Install miss entry pointing to group
 747         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 748         mdsalManager.addFlow(confTx, flowEntity);
 749     }
 750
 751     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 752             String routerName, long routerId) {
 753         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 754             dpnId, bucketInfo.get(0));
 755         // Install the select group
 756         long groupId = createGroupId(getGroupIdKey(routerName));
 757         GroupEntity groupEntity =
 758             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 759         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 760         mdsalManager.syncInstallGroup(groupEntity);
 761         // Install miss entry pointing to group
 762         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 763         if (flowEntity == null) {
 764             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 765                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 766                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 767             return;
 768         }
 769         mdsalManager.installFlow(flowEntity);
 770     }
 771
 772     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 773         long groupId = createGroupId(getGroupIdKey(routerName));
 774         GroupEntity groupEntity =
 775             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 776         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 777         mdsalManager.syncInstallGroup(groupEntity);
 778         return groupId;
 779     }
 780
 781     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 782         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 783                 dpId, routerName, groupId);
 784         List<MatchInfo> matches = new ArrayList<>();
 785         matches.add(MatchEthernetType.IPV4);
 786         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 787
 788         List<ActionInfo> actionsInfo = new ArrayList<>();
 789         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 790                 routerId, routerName);
 791         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 792         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 793         actionsInfo.add(new ActionGroup(groupId));
 794         List<InstructionInfo> instructions = new ArrayList<>();
 795         instructions.add(new InstructionApplyActions(actionsInfo));
 796         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 797         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 798                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 799                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 800
 801         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 802         return flowEntity;
 803     }
 804
 805     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 806
 807         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 808             routerName);
 809         List<MatchInfo> matches = new ArrayList<>();
 810         matches.add(MatchEthernetType.IPV4);
 811         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 812
 813         List<InstructionInfo> instructions = new ArrayList<>();
 814         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 815
 816         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 817         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 818             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 819             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 820
 821         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 822         return flowEntity;
 823     }
 824
 825     // TODO : Replace this with ITM Rpc once its available with full functionality
 826     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 827         TypedWriteTransaction<Configuration> confTx) {
 828
 829         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 830             dpnId, routerName);
 831         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 832         if (flowEntity == null) {
 833             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 834                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 835                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 836                     dpnId, routerName);
 837             return;
 838         }
 839         mdsalManager.addFlow(confTx, flowEntity);
 840
 841     }
 842
 843     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 844         List<MatchInfo> matches = new ArrayList<>();
 845         matches.add(MatchEthernetType.IPV4);
 846         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 847                 idManager, routerId, routerName);
 848         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 849         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 850         List<InstructionInfo> instructions = new ArrayList<>();
 851         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 852                 MetaDataUtil.METADATA_MASK_VRFID));
 853         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 854         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 855                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 856                 instructions);
 857         return flowEntity;
 858     }
 859
 860     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 861         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 862                 .FLOWID_SEPARATOR + routerID;
 863     }
 864
 865     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 866         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 867                 .FLOWID_SEPARATOR + routerID;
 868     }
 869
 870     private String getGroupIdKey(String routerName) {
 871         return "snatmiss." + routerName;
 872     }
 873
 874     protected long createGroupId(String groupIdKey) {
 875         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 876             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 877             .build();
 878         try {
 879             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 880             RpcResult<AllocateIdOutput> rpcResult = result.get();
 881             return rpcResult.getResult().getIdValue();
 882         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 883             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 884         }
 885         return 0;
 886     }
 887
 888     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 889         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 890         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 891         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 892         List<BucketInfo> listBucketInfo = new ArrayList<>();
 893
 894         if (ifNamePrimary != null) {
 895             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 896             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 897                     interfaceManager, ifNamePrimary, routerId, true);
 898             if (listActionInfoPrimary.isEmpty()) {
 899                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 900                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 901                         ifNamePrimary);
 902             }
 903         } else {
 904             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 905                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 906         }
 907         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 908
 909         listBucketInfo.add(0, bucketPrimary);
 910         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 911     }
 912
 913     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 914                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 915         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 916         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 917         List<BucketInfo> listBucketInfo = new ArrayList<>();
 918
 919         if (ifNamePrimary != null) {
 920             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 921                     ifNamePrimary);
 922             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 923                     interfaceManager, ifNamePrimary, routerId, true);
 924             if (listActionInfoPrimary.isEmpty()) {
 925                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 926                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 927                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 928             }
 929         } else {
 930             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 931                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 932         }
 933         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 934
 935         listBucketInfo.add(0, bucketPrimary);
 936         return listBucketInfo;
 937     }
 938
 939     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 940         TypedWriteTransaction<Configuration> confTx) {
 941
 942        /*
 943         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 944         */
 945
 946         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 947
 948 /*
 949         List<BucketInfo> listBucketInfo = new ArrayList<>();
 950         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 951         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 952         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 953         listBucketInfo.add(0, bucketPrimary);
 954 */
 955
 956         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 957         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 958         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 959         installNaptPfibEntry(dpnId, routerId, confTx);
 960         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 961         if (networkId != null) {
 962             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 963             if (vpnUuid != null) {
 964                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 965                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 966                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 967                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 968                     if (vpnId != NatConstants.INVALID_ID) {
 969                         installNaptPfibEntry(dpnId, vpnId, null);
 970                     }
 971                     return Collections.emptyList();
 972                 });
 973             } else {
 974                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 975             }
 976         } else {
 977             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 978         }
 979     }
 980
 981     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
 982             @Nullable TypedWriteTransaction<Configuration> confTx) {
 983         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 984         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 985         if (confTx != null) {
 986             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 987         } else {
 988             mdsalManager.installFlow(naptPfibFlowEntity);
 989         }
 990     }
 991
 992     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 993
 994         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 995         List<MatchInfo> matches = new ArrayList<>();
 996         matches.add(MatchEthernetType.IPV4);
 997         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 998
 999         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1000         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1001         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1002         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1003         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1004
1005         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1006         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1007             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1008             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1009         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1010         return flowEntity;
1011     }
1012
1013     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1014         long routerId, String routerName, String externalIp) {
1015         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1016             dpnId, routerId, externalIp);
1017         Uuid networkId = router.getNetworkId();
1018         if (networkId == null) {
1019             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1020             return;
1021         }
1022         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1023         if (vpnName == null) {
1024             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1025                     + "configuration {} in router {}", networkId, externalIp, routerId);
1026             return;
1027         }
1028         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1029             externalIp, networkId, router, confTx);
1030         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1031             dpnId, routerId, externalIp);
1032     }
1033
1034     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1035                                                 final long routerId, final String routerName, final String externalIp,
1036                                                 final Uuid extNetworkId, @Nullable final Routers router,
1037                                                 final TypedWriteTransaction<Configuration> confTx) {
1038         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1039                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1040         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1041         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1042         if (rd == null || rd.isEmpty()) {
1043             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1044             return;
1045         }
1046         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1047         if (extNwProvType == null) {
1048             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1049             return;
1050         }
1051         if (extNwProvType == ProviderTypes.VXLAN) {
1052             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1053                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1054             return;
1055         }
1056         //Generate VPN label for the external IP
1057         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1058             .setIpPrefix(externalIp).build();
1059         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1060
1061         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1062         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1063             if (result.isSuccessful()) {
1064                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1065                 GenerateVpnLabelOutput output = result.getResult();
1066                 final long label = output.getLabel();
1067
1068                 int externalIpInDsFlag = 0;
1069                 //Get IPMaps from the DB for the router ID
1070                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1071                 for (IpMap dbIpMap : dbIpMaps) {
1072                     String dbExternalIp = dbIpMap.getExternalIp();
1073                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1074                     if (dbExternalIp.contains(externalIp)) {
1075                         String dbInternalIp = dbIpMap.getInternalIp();
1076                         IpMapKey dbIpMapKey = dbIpMap.key();
1077                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1078                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1079                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1080                             .setExternalIp(dbExternalIp).setLabel(label).build();
1081                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1082                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1083                         externalIpInDsFlag++;
1084                     }
1085                 }
1086                 if (externalIpInDsFlag <= 0) {
1087                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1088                             + "Failed to update label {} for routerId {} in DS",
1089                             externalIp, label, routerId);
1090                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1091                         + " found in DS for router %s", label, externalIp, routerId);
1092                     return Futures.immediateFailedFuture(new Exception(errMsg));
1093                 }
1094                 //Inform BGP
1095                 long l3vni = 0;
1096                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1097                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1098                 }
1099                 Routers extRouter = router != null ? router :
1100                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1101                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1102                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1103                     RouteOrigin.STATIC, dpnId);
1104
1105                 //Install custom FIB routes
1106                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1107                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1108                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1109                         extNwProvType);
1110                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1111
1112                 //Install custom FIB routes - FIB table.
1113                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1114                         routerName, externalIp);
1115                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1116                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1117                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1118                 }
1119                 String externalVpn = vpnName;
1120                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1121                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1122                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1123                             externalSubnetId);
1124                     if (externalSubnet.isPresent()) {
1125                         externalVpn =  externalSubnetId.getValue();
1126                     }
1127                 }
1128                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1129                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1130                     .setVpnName(externalVpn)
1131                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1132                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1133                     .setInstruction(fibTableCustomInstructions).build();
1134                 return fibService.createFibEntry(input);
1135             } else {
1136                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1137                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1138                     externalIp, vpnName, result.getErrors());
1139                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1140             }
1141         }, MoreExecutors.directExecutor());
1142
1143         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1144
1145             @Override
1146             public void onFailure(@NonNull Throwable error) {
1147                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1148             }
1149
1150             @Override
1151             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1152                 if (result.isSuccessful()) {
1153                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1154                             externalIp);
1155                 } else {
1156                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1157                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1158                 }
1159             }
1160         }, MoreExecutors.directExecutor());
1161     }
1162
1163     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1164             String externalIp) {
1165         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1166         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1167         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1168                 externalIp, router);
1169         int instructionIndex = 0;
1170         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1171             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1172             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1173                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1174             instructionIndex++;
1175         }
1176
1177         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1178         return fibTableCustomInstructions;
1179     }
1180
1181     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1182         TypedWriteTransaction<Configuration> confTx) {
1183         List<MatchInfo> matches = new ArrayList<>();
1184         matches.add(MatchEthernetType.MPLS_UNICAST);
1185         matches.add(new MatchMplsLabel(serviceId));
1186
1187         List<Instruction> instructions = new ArrayList<>();
1188         List<ActionInfo> actionsInfos = new ArrayList<>();
1189         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1190         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1191         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1192         instructions.add(writeInstruction);
1193         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1194
1195         // Install the flow entry in L3_LFIB_TABLE
1196         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1197
1198         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1199             10, flowRef, 0, 0,
1200             COOKIE_VM_LFIB_TABLE, matches, instructions);
1201
1202         mdsalManager.addFlow(confTx, dpId, flowEntity);
1203
1204         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1205     }
1206
1207     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1208         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1209         ProviderTypes extNwProvType) {
1210
1211         List<MatchInfo> mkMatches = new ArrayList<>();
1212
1213         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1214             dpnId, serviceId, customInstructions);
1215
1216         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1217             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1218         } else {
1219             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1220         }
1221
1222         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1223             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1224             String.format("%s:%d", "TST Flow Entry ", serviceId),
1225             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1226
1227         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1228     }
1229
1230     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1231         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1232             new RouterIdsKey(routerId)).build();
1233     }
1234
1235     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1236         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1237                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1238     }
1239
1240     @Override
1241     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1242         String routerName = original.getRouterName();
1243         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1244         if (routerId == NatConstants.INVALID_ID) {
1245             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1246             return;
1247         }
1248         // Check if its update on SNAT flag
1249         boolean originalSNATEnabled = original.isEnableSnat();
1250         boolean updatedSNATEnabled = update.isEnableSnat();
1251         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1252             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1253         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1254          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1255          */
1256         long bgpVpnId = NatConstants.INVALID_ID;
1257         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1258         if (bgpVpnUuid != null) {
1259             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1260         }
1261         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1262         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1263             // Router has no interface attached
1264             return;
1265         }
1266         final long finalBgpVpnId = bgpVpnId;
1267         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1268             List<ListenableFuture<Void>> futures = new ArrayList<>();
1269             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1270                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1271                     Uuid networkId = original.getNetworkId();
1272                     if (originalSNATEnabled != updatedSNATEnabled) {
1273                         if (originalSNATEnabled) {
1274                             //SNAT disabled for the router
1275                             Uuid networkUuid = original.getNetworkId();
1276                             LOG.info("update : SNAT disabled for Router {}", routerName);
1277                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1278                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1279                                     removeFlowInvTx);
1280                         } else {
1281                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1282                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1283                                     true, writeFlowInvTx);
1284                             handleEnableSnat(update, routerId, dpnId, finalBgpVpnId, writeFlowInvTx);
1285                         }
1286                     }
1287                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1288                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1289                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1290                     }
1291
1292                     if (updatedSNATEnabled != originalSNATEnabled) {
1293                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1294                                 + "handleDisableSnat/handleEnableSnat");
1295                         return;
1296                     }
1297                     //Check if the Update is on External IPs
1298                     LOG.debug("update : Checking if this is update on External IPs");
1299                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1300                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1301
1302                     //Check if the External IPs are removed during the update.
1303                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1304                     removedExternalIps.removeAll(updatedExternalIps);
1305                     if (removedExternalIps.size() > 0) {
1306                         LOG.debug("update : Start processing of the External IPs removal during the update "
1307                                 + "operation");
1308                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1309                                 removedExternalIps, original.getExtGwMacAddress(),
1310                                 dpnId, networkId);
1311
1312                         for (String removedExternalIp : removedExternalIps) {
1313                 /*
1314                     1) Remove the mappings in the IntExt IP model which has external IP.
1315                     2) Remove the external IP in the ExternalCounter model.
1316                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1317                        least loaded external IP.
1318                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1319                     4) Increase the count of the allocated external IP by one.
1320                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1321                      i.e. the route for the external IP is absent.
1322                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1323                      the removed external IPs and also from the model.
1324                     7) Advertise to the BGP for removing the route for the removed external IPs.
1325                  */
1326
1327                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1328                             String externalIp = externalIpParts[0];
1329                             String externalIpPrefix = externalIpParts[1];
1330                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1331
1332                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1333                                     + "entries for removed external IP {}", externalIpAddrStr);
1334                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1335                             String vpnName = "";
1336                             if (vpnUuId != null) {
1337                                 vpnName = vpnUuId.getValue();
1338                             }
1339                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1340                                     update.getExtGwMacAddress(), removeFlowInvTx);
1341
1342                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1343                             //Get the internal IPs which are associated to the removed external IPs
1344                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1345                             List<String> removedInternalIps = new ArrayList<>();
1346                             for (IpMap ipMap : ipMaps) {
1347                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1348                                     removedInternalIps.add(ipMap.getInternalIp());
1349                                 }
1350                             }
1351
1352                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1353                             for (String removedInternalIp : removedInternalIps) {
1354                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1355                                                 + "router ID {} from the IntExtIP model",
1356                                         removedInternalIp, routerId);
1357                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1358                             }
1359
1360                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1361                                             + "router ID {} from the ExternalIpsCounter model.",
1362                                     externalIpAddrStr, routerId);
1363                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1364
1365                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1366                                     + "whose external IPs were removed.");
1367                             for (String removedInternalIp : removedInternalIps) {
1368                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1369                                         removedInternalIp, writeFlowInvTx);
1370                             }
1371                             LOG.debug("update : Remove the NAPT translation entries from "
1372                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1373                             //Get the internalIP and internal Port which were associated to the removed external IP.
1374                             List<Integer> externalPorts = new ArrayList<>();
1375                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1376                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1377                                     .builder(IntextIpPortMap.class)
1378                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1379                             Optional<IpPortMapping> ipPortMapping;
1380                             try {
1381                                 ipPortMapping = SingleTransactionDataBroker
1382                                             .syncReadOptional(dataBroker,
1383                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1384                             } catch (ReadFailedException e) {
1385                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1386                                 ipPortMapping = Optional.absent();
1387                             }
1388
1389                             if (ipPortMapping.isPresent()) {
1390                                 for (IntextIpProtocolType intextIpProtocolType :
1391                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1392                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1393                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1394                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1395                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1396                                             externalPorts.add(ipPortExternal.getPortNum());
1397                                             List<String> removedInternalIpPorts =
1398                                                     protoTypesIntIpPortsMap.get(protoType);
1399                                             if (removedInternalIpPorts != null) {
1400                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1401                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1402                                             } else {
1403                                                 removedInternalIpPorts = new ArrayList<>();
1404                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1405                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1406                                             }
1407                                         }
1408                                     }
1409                                 }
1410                             }
1411
1412                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1413                             // the removed external IP.
1414                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1415                                     protoTypesIntIpPortsMap.entrySet();
1416                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1417                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1418                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1419                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1420                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1421                                     // Remove the IP port map from the intext-ip-port-map model,
1422                                     // which were containing the removed external IP
1423                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1424                                             protocolType);
1425                                     //Remove the IP port incomint packer map.
1426                                     naptPacketInHandler.removeIncomingPacketMap(
1427                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1428                                     String[] removedInternalIpPortParts = removedInternalIpPort
1429                                             .split(NatConstants.COLON_SEPARATOR);
1430                                     if (removedInternalIpPortParts.length == 2) {
1431                                         String removedInternalIp = removedInternalIpPortParts[0];
1432                                         String removedInternalPort = removedInternalIpPortParts[1];
1433                                         List<String> removedInternalPortsList =
1434                                                 internalIpPortMap.get(removedInternalPort);
1435                                         if (removedInternalPortsList != null) {
1436                                             removedInternalPortsList.add(removedInternalPort);
1437                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1438                                         } else {
1439                                             removedInternalPortsList = new ArrayList<>();
1440                                             removedInternalPortsList.add(removedInternalPort);
1441                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1442                                         }
1443                                     }
1444                                 }
1445                             }
1446
1447                             // Delete the entry from SnatIntIpPortMap DS
1448                             Set<String> internalIps = internalIpPortMap.keySet();
1449                             for (String internalIp : internalIps) {
1450                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1451                                         + "model SnatIntIpPortMap", internalIp);
1452                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1453                             }
1454
1455                             naptManager.removeNaptPortPool(externalIp);
1456
1457                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1458                                     + "the removed external IP {}", externalIp);
1459                             for (Integer externalPort : externalPorts) {
1460                                 //Remove the NAPT translation entries from Inbound NAPT table
1461                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1462                                         routerId, externalIp, externalPort);
1463                             }
1464
1465                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1466                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1467                                 String internalIp = internalIpPort.getKey();
1468                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1469                                         + "for the removed internal IP {}", internalIp);
1470                                 List<String> internalPorts = internalIpPort.getValue();
1471                                 for (String internalPort : internalPorts) {
1472                                     //Remove the NAPT translation entries from Outbound NAPT table
1473                                     naptPacketInHandler.removeIncomingPacketMap(
1474                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1475                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1476                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1477                                             routerId, internalIp, Integer.parseInt(internalPort));
1478                                 }
1479                             }
1480                         }
1481                         LOG.debug(
1482                                 "update : End processing of the External IPs removal during the update operation");
1483                     }
1484
1485                     //Check if the External IPs are added during the update.
1486                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1487                     addedExternalIps.removeAll(originalExternalIps);
1488                     if (addedExternalIps.size() != 0) {
1489                         LOG.debug("update : Start processing of the External IPs addition during the update "
1490                                 + "operation");
1491                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1492                                 update.getExtGwMacAddress(), dpnId,
1493                                 update.getNetworkId());
1494
1495                         for (String addedExternalIp : addedExternalIps) {
1496                 /*
1497                     1) Do nothing in the IntExtIp model.
1498                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1499                  */
1500                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1501                             String externalIp = externalIpParts[0];
1502                             String externalIpPrefix = externalIpParts[1];
1503                             String externalpStr = externalIp + "/" + externalIpPrefix;
1504                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1505                                             + "router ID {} in the ExternalIpsCounter model.",
1506                                     externalpStr, routerId);
1507                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1508                             subnetRegisterMapping(update, routerId);
1509                             LOG.info("update : Installing fib flow fo newly added Ips");
1510                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1511                         }
1512                         LOG.debug(
1513                                 "update : End processing of the External IPs addition during the update operation");
1514                     }
1515
1516                     //Check if its Update on subnets
1517                     LOG.debug("update : Checking if this is update on subnets");
1518                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1519                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1520                     Set<Uuid> addedSubnetIds =
1521                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1522                     if (originalSubnetIds != null) {
1523                         addedSubnetIds.removeAll(originalSubnetIds);
1524                     }
1525
1526                     //Check if the Subnet IDs are added during the update.
1527                     if (addedSubnetIds.size() != 0) {
1528                         LOG.debug(
1529                                 "update : Start processing of the Subnet IDs addition during the update operation");
1530                         for (Uuid addedSubnetId : addedSubnetIds) {
1531                 /*
1532                     1) Select the least loaded external IP for the subnet and store the mapping of the
1533                     subnet IP and the external IP in the IntExtIp model.
1534                     2) Increase the count of the selected external IP by one.
1535                     3) Advertise to the BGP if external IP is allocated for the first time for the
1536                     router i.e. the route for the external IP is absent.
1537                  */
1538                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1539                             if (subnetIp != null) {
1540                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1541                                         writeFlowInvTx);
1542                             }
1543                         }
1544                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1545                     }
1546
1547                     //Check if the Subnet IDs are removed during the update.
1548                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1549                     removedSubnetIds.removeAll(updatedSubnetIds);
1550                     if (removedSubnetIds.size() != 0) {
1551                         LOG.debug(
1552                                 "update : Start processing of the Subnet IDs removal during the update operation");
1553                         for (Uuid removedSubnetId : removedSubnetIds) {
1554                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1555                             if (subnetAddr != null) {
1556                     /*
1557                         1) Remove the subnet IP and the external IP in the IntExtIp map
1558                         2) Decrease the count of the coresponding external IP by one.
1559                         3) Advertise to the BGP for removing the routes of the corresponding external
1560                         IP if its not allocated to any other internal IP.
1561                     */
1562
1563                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1564                                         subnetAddr[0] + "/" + subnetAddr[1]);
1565                                 if (externalIp == null) {
1566                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1567                                             routerId, subnetAddr[0]);
1568                                     return;
1569                                 }
1570
1571                                 naptManager.updateCounter(routerId, externalIp, false);
1572                                 // Traverse entire model of external-ip counter whether external ip is not
1573                                 // used by any other internal ip in any router
1574                                 if (!isExternalIpAllocated(externalIp)) {
1575                                     LOG.debug("update : external ip is not allocated to any other "
1576                                             + "internal IP so proceeding to remove routes");
1577                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1578                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1579                                             removeFlowInvTx);
1580                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1581                                                     + "router {} with networkId {} and externalIp {}",
1582                                             dpnId, routerId, networkId, externalIp);
1583                                 }
1584
1585                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1586                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1587                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1588                             }
1589                         }
1590                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1591                     }
1592                 }));
1593             }));
1594             return futures;
1595         }, NatConstants.NAT_DJC_MAX_RETRIES);
1596     }
1597
1598     private boolean isExternalIpAllocated(String externalIp) {
1599         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1600         Optional<ExternalIpsCounter> externalCountersData;
1601         try {
1602             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1603                         LogicalDatastoreType.OPERATIONAL, id);
1604         } catch (ReadFailedException e) {
1605             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1606             externalCountersData = Optional.absent();
1607         }
1608         if (externalCountersData.isPresent()) {
1609             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1610             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1611                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1612                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1613                         if (externalIpCount.getCounter() != 0) {
1614                             return true;
1615                         }
1616                         break;
1617                     }
1618                 }
1619             }
1620         }
1621         return false;
1622     }
1623
1624     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1625             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1626         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1627         try {
1628             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1629             if (address instanceof Inet6Address) {
1630                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1631                 return;
1632             }
1633         } catch (UnknownHostException e) {
1634             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1635             return;
1636         }
1637         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1638         if (leastLoadedExtIpAddr != null) {
1639             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1640             String leastLoadedExtIp = externalIpParts[0];
1641             String leastLoadedExtIpPrefix = externalIpParts[1];
1642             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1643             subnetIp = subnetIpParts[0];
1644             String subnetIpPrefix = subnetIpParts[1];
1645             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1646             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1647                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1648                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1649             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1650
1651
1652             // Check if external IP is already assigned a route. (i.e. External IP is previously
1653             // allocated to any of the subnets)
1654             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1655             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1656             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1657             if (label != null) {
1658                 //update
1659                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1660                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1661                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1662                     label, internalIp, leastLoadedExtIpAddrStr);
1663                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1664                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1665                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1666                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1667                 return;
1668             }
1669
1670             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1671             // for the first time and hence not having a route.
1672             //Get the VPN Name using the network ID
1673             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1674             if (vpnName != null) {
1675                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1676                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1677                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1678                         + "added after gateway-set");
1679                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1680                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1681                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1682                         return;
1683                     }
1684                 }
1685                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1686                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1687                     writeFlowInvTx);
1688             }
1689         }
1690     }
1691
1692     @Nullable
1693     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1694         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1695         for (IpMap ipMap : ipMaps) {
1696             if (ipMap.getExternalIp().equals(externalIp)) {
1697                 if (ipMap.getLabel() != null) {
1698                     return ipMap.getLabel();
1699                 }
1700             }
1701         }
1702         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1703         return null;
1704     }
1705
1706     @Override
1707     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1708         LOG.trace("remove : Router delete method");
1709         /*
1710         ROUTER DELETE SCENARIO
1711         1) Get the router ID from the event.
1712         2) Build the cookie information from the router ID.
1713         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1714         4) Build the flow with the cookie value.
1715         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1716         6) Remove the flows from the other switches which points to the primary and secondary
1717          switches for the flows related the router ID.
1718         7) Get the list of external IP address maintained for the router ID.
1719         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1720         9) Withdraw the corresponding routes from the BGP.
1721          */
1722
1723         if (identifier == null || router == null) {
1724             LOG.error("remove : returning without processing since routers is null");
1725             return;
1726         }
1727
1728         String routerName = router.getRouterName();
1729         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1730             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1731                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1732                         routerName);
1733                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1734                 if (routerId == NatConstants.INVALID_ID) {
1735                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1736                             routerName);
1737                     return;
1738                 }
1739                 long bgpVpnId = NatConstants.INVALID_ID;
1740                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1741                 if (bgpVpnUuid != null) {
1742                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1743                 }
1744                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1745                         tx);
1746                 Uuid networkUuid = router.getNetworkId();
1747
1748                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1749                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1750                     // No NAPT switch for external router, probably because the router is not attached to
1751                     // any
1752                     // internal networks
1753                     LOG.debug(
1754                             "No NAPT switch for router {}, check if router is attached to any internal "
1755                                     + "network",
1756                             routerName);
1757                     return;
1758                 } else {
1759                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1760                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1761                             routerId, tx);
1762                 }
1763                 natOverVxlanUtil.releaseVNI(routerName);
1764             })), NatConstants.NAT_DJC_MAX_RETRIES);
1765     }
1766
1767     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1768                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1769                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1770         LOG.info("handleDisableSnat : Entry");
1771         String routerName = router.getRouterName();
1772         try {
1773             if (routerFlag) {
1774                 removeNaptSwitch(routerName);
1775             } else {
1776                 updateNaptSwitch(routerName, BigInteger.ZERO);
1777             }
1778
1779             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1780             naptManager.removeExternalCounter(routerId);
1781
1782             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1783             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1784                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1785                     + "router ID {} from RouterNaptSwitch model", routerId);
1786                 return;
1787             }
1788             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1789                     networkUuid);
1790             if (extNwProvType == null) {
1791                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1792                 return;
1793             }
1794             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1795             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1796                     externalSubnetList, removeFlowInvTx, extNwProvType);
1797             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1798             String externalSubnetVpn = null;
1799             for (Uuid externalSubnetId : externalSubnetList) {
1800                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1801                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1802                 if (externalSubnet.isPresent()) {
1803                     externalSubnetVpn =  externalSubnetId.getValue();
1804                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1805                             router.getExtGwMacAddress(), removeFlowInvTx);
1806                 }
1807             }
1808             if (externalSubnetVpn == null) {
1809                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1810                         router.getExtGwMacAddress(), removeFlowInvTx);
1811             }
1812             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1813             // for the router ID.
1814             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1815                     + "router ID {} in the DS", routerId);
1816             naptManager.removeMapping(routerId);
1817         } catch (InterruptedException | ExecutionException e) {
1818             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1819         }
1820         LOG.info("handleDisableSnat : Exit");
1821     }
1822
1823     // TODO Clean up the exception handling
1824     @SuppressWarnings("checkstyle:IllegalCatch")
1825     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1826                                              @NonNull Collection<String> externalIps,
1827                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1828         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1829                 + "with internet vpn {}", routerName, vpnId);
1830         try {
1831             BigInteger naptSwitchDpnId = null;
1832             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1833                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1834             Optional<RouterToNaptSwitch> rtrToNapt;
1835             try {
1836                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1837                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1838             } catch (ReadFailedException e) {
1839                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1840                 rtrToNapt = Optional.absent();
1841             }
1842             if (rtrToNapt.isPresent()) {
1843                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1844             }
1845             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1846
1847             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1848                     writeFlowInvTx);
1849             try {
1850                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1851                 if (extGwMacAddress != null) {
1852                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1853                             + "External Router ID {}", extGwMacAddress, routerId);
1854                 } else {
1855                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1856                             + "External Router ID {}", routerId);
1857                     return;
1858                 }
1859                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1860                         writeFlowInvTx);
1861             } catch (Exception ex) {
1862                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1863                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1864             }
1865             natOverVxlanUtil.releaseVNI(vpnId);
1866         } catch (InterruptedException | ExecutionException e) {
1867             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1868                     + "with internet vpn {}", routerName, vpnId, e);
1869         }
1870         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1871                 routerName, vpnId);
1872     }
1873
1874     // TODO Clean up the exception handling
1875     @SuppressWarnings("checkstyle:IllegalCatch")
1876     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1877         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1878             .setPrimarySwitchId(naptSwitchId).build();
1879         try {
1880             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1881                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1882         } catch (Exception ex) {
1883             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1884                 naptSwitchId, routerName);
1885         }
1886         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1887             naptSwitchId, routerName);
1888     }
1889
1890     protected void removeNaptSwitch(String routerName) {
1891         // Remove router and switch from model
1892         InstanceIdentifier<RouterToNaptSwitch> id =
1893             InstanceIdentifier.builder(NaptSwitches.class)
1894                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1895         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1896         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1897         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1898         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1899     }
1900
1901     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1902                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1903                                                 @NonNull Collection<String> externalIps,
1904                                                 Collection<Uuid> externalSubnetList,
1905                                                 TypedReadWriteTransaction<Configuration> confTx,
1906                                                 ProviderTypes extNwProvType)
1907             throws InterruptedException, ExecutionException {
1908
1909         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1910         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1911
1912         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1913         // traffic which comes from the  VMs of the NAPT switches)
1914         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1915         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1916
1917         LOG.info(
1918             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1919                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1920         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1921
1922         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1923         // traffic which comes from the VMs of the non NAPT switches)
1924         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1925                 elanManager, idManager, routerId, routerName);
1926         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1927         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1928         LOG.info(
1929             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1930                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1931         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1932
1933         //Remove the flow table 25->44 from NAPT Switch
1934         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1935             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1936         }
1937
1938         //Remove the Outbound flow entry which forwards the packet to FIB Table
1939         LOG.info(
1940             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1941                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1942
1943         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1944             NwConstants.IP_PROT_TCP);
1945         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1946             outboundTcpNatFlowRef);
1947         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1948
1949         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1950             NwConstants.IP_PROT_UDP);
1951         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1952             outboundUdpNatFlowRef);
1953         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1954
1955         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1956             NwConstants.IP_PROT_ICMP);
1957         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1958             icmpDropFlowRef);
1959         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1960         boolean lastRouterOnExternalNetwork =
1961             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1962         if (lastRouterOnExternalNetwork) {
1963             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1964         }
1965         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1966         // External Subnet Vpn Id.
1967         for (Uuid externalSubnetId : externalSubnetList) {
1968             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1969             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1970                 dataBroker, externalSubnetId, routerName, dpnId)) {
1971                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1972                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1973                     natPfibSubnetFlowRef);
1974                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1975                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1976                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1977                     subnetVpnId, dpnId);
1978             }
1979         }
1980
1981         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1982         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1983         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1984
1985         LOG.info(
1986             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1987             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1988         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1989
1990         if (lastRouterOnExternalNetwork) {
1991             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1992             // - This does not work since ext-routers is deleted already - no network info
1993             //Get the VPN ID from the ExternalNetworks model
1994             long vpnId = -1;
1995             if (vpnName == null || vpnName.isEmpty()) {
1996                 // ie called from router delete cases
1997                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1998                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1999                 if (vpnUuid != null) {
2000                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2001                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2002                               + "or disableSNAT scenario", vpnId, networkId);
2003                 }
2004             } else {
2005                 // ie called from disassociate vpn case
2006                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2007                     vpnName);
2008                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2009                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2010                     vpnId);
2011             }
2012
2013             if (vpnId != NatConstants.INVALID_ID) {
2014                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2015                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2016                 FlowEntity natPfibVpnFlowEntity =
2017                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2018                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2019                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2020                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2021             }
2022         }
2023
2024         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2025         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2026         if (ipPortMapping == null) {
2027             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2028             return;
2029         }
2030
2031         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2032             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2033                 String ipPortInternal = ipPortMap.getIpPortInternal();
2034                 String[] ipPortParts = ipPortInternal.split(":");
2035                 if (ipPortParts.length != 2) {
2036                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2037                     return;
2038                 }
2039                 String internalIp = ipPortParts[0];
2040                 String internalPort = ipPortParts[1];
2041
2042                 //Build the flow for the outbound NAPT table
2043                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2044                     + NatConstants.COLON_SEPARATOR + internalPort);
2045                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2046                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2047                 FlowEntity outboundNaptFlowEntity =
2048                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2049
2050                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2051                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2052                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2053
2054                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2055                 String externalIp = ipPortExternal.getIpAddress();
2056                 int externalPort = ipPortExternal.getPortNum();
2057
2058                 //Build the flow for the inbound NAPT table
2059                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2060                     String.valueOf(routerId), externalIp, externalPort);
2061                 FlowEntity inboundNaptFlowEntity =
2062                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2063
2064                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2065                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2066                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2067             }
2068         }
2069     }
2070
2071     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2072                                                     @NonNull Collection<String> externalIps,
2073                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2074             throws ExecutionException, InterruptedException {
2075         long extVpnId = NatConstants.INVALID_ID;
2076         if (networkId != null) {
2077             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2078             if (vpnUuid != null) {
2079                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2080             } else {
2081                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2082             }
2083         } else {
2084             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2085             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2086         }
2087         if (extVpnId == NatConstants.INVALID_ID) {
2088             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2089             extVpnId = routerId;
2090         }
2091         for (String ip : externalIps) {
2092             String extIp = removeMaskFromIp(ip);
2093             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2094             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2095                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2096                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2097             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2098             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2099         }
2100     }
2101
2102     private String removeMaskFromIp(String ip) {
2103         if (ip != null && !ip.trim().isEmpty()) {
2104             return ip.split("/")[0];
2105         }
2106         return ip;
2107     }
2108
2109     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2110                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2111                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2112             throws ExecutionException, InterruptedException {
2113         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2114         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2115
2116         //Remove the NAPT PFIB TABLE entry
2117         long vpnId = -1;
2118         if (vpnName != null) {
2119             // ie called from disassociate vpn case
2120             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2121                     + "with vpnName {}", vpnName);
2122             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2123             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2124                     vpnId);
2125         }
2126
2127         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2128                 networkId, routerName, dpnId)) {
2129             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2130             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2131             FlowEntity natPfibVpnFlowEntity =
2132                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2133             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2134                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2135             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2136
2137             // Remove IP-PORT active NAPT entries and release port from IdManager
2138             // For the router ID get the internal IP , internal port and the corresponding
2139             // external IP and external Port.
2140             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2141             if (ipPortMapping == null) {
2142                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2143                 return;
2144             }
2145             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2146                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2147                     String ipPortInternal = ipPortMap.getIpPortInternal();
2148                     String[] ipPortParts = ipPortInternal.split(":");
2149                     if (ipPortParts.length != 2) {
2150                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2151                                 + "and port");
2152                         return;
2153                     }
2154                     String internalIp = ipPortParts[0];
2155                     String internalPort = ipPortParts[1];
2156
2157                     //Build the flow for the outbound NAPT table
2158                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2159                             + NatConstants.COLON_SEPARATOR + internalPort);
2160                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2161                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2162                     FlowEntity outboundNaptFlowEntity =
2163                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2164
2165                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2166                             + "active switch with the DPN ID {} and router ID {}",
2167                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2168                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2169
2170                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2171                     String externalIp = ipPortExternal.getIpAddress();
2172                     int externalPort = ipPortExternal.getPortNum();
2173
2174                     //Build the flow for the inbound NAPT table
2175                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2176                         String.valueOf(routerId), externalIp, externalPort);
2177                     FlowEntity inboundNaptFlowEntity =
2178                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2179
2180                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2181                             + "active active switch with the DPN ID {} and router ID {}",
2182                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2183                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2184
2185                     // Finally release port from idmanager
2186                     String internalIpPort = internalIp + ":" + internalPort;
2187                     naptManager.removePortFromPool(internalIpPort, externalIp);
2188
2189                     //Remove sessions from models
2190                     naptManager.removeIpPortMappingForRouterID(routerId);
2191                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2192                 }
2193             }
2194         } else {
2195             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2196         }
2197     }
2198
2199     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2200             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2201             throws ExecutionException, InterruptedException {
2202         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2203
2204         // Remove the flows from the other switches which points to the primary and secondary switches
2205         // for the flows related the router ID.
2206         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2207         if (allSwitchList.isEmpty()) {
2208             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2209             return;
2210         }
2211         for (BigInteger dpnId : allSwitchList) {
2212             if (!naptSwitchDpnId.equals(dpnId)) {
2213                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2214
2215                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2216                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2217                 FlowEntity preSnatFlowEntity =
2218                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2219
2220                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2221                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2222                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2223
2224                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2225                 long groupId = createGroupId(getGroupIdKey(routerName));
2226
2227                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2228                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2229                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2230             }
2231         }
2232     }
2233
2234     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2235                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2236                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2237             throws ExecutionException, InterruptedException {
2238         //Withdraw the corresponding routes from the BGP.
2239         //Get the network ID using the router ID.
2240         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2241                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2242         if (networkUuid == null) {
2243             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2244             return;
2245         }
2246
2247         if (externalIps.isEmpty()) {
2248             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2249             return;
2250         }
2251
2252         if (vpnName == null) {
2253             //Get the VPN Name using the network ID
2254             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2255             if (vpnName == null) {
2256                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2257                     networkUuid, routerId);
2258                 return;
2259             }
2260         }
2261         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2262
2263         //Remove custom FIB routes
2264         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2265         for (String extIp : externalIps) {
2266             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2267         }
2268     }
2269
2270     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2271                                             final Uuid networkUuid, String extGwMacAddress,
2272                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2273             throws ExecutionException, InterruptedException {
2274         clearBgpRoutes(extIp, vpnName);
2275         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2276                 removeFlowInvTx);
2277     }
2278
2279     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2280                                              String vpnName, Uuid extNetworkId, long tempLabel,
2281                                              String gwMacAddress, boolean switchOver,
2282                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2283             throws ExecutionException, InterruptedException {
2284         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2285         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2286         if (routerName == null) {
2287             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2288             return;
2289         }
2290         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2291         if (extNwProvType == null) {
2292             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2293             return;
2294         }
2295         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2296          * external network provided type is VxLAN
2297          */
2298         if (extNwProvType == ProviderTypes.VXLAN) {
2299             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2300             );
2301             return;
2302         }
2303         if (tempLabel < 0) {
2304             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2305                     extIp, routerId);
2306             return;
2307         }
2308         final long label = tempLabel;
2309         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2310         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2311             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2312             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2313             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2314                     extRouter);
2315
2316             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2317                     externalSubnetId);
2318
2319             if (externalSubnet.isPresent()) {
2320                 vpnName =  externalSubnetId.getValue();
2321             }
2322         }
2323         final String externalVpn = vpnName;
2324         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2325                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2326                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2327         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2328
2329         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2330         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2331         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2332             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2333             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2334         }
2335         if (!switchOver) {
2336             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2337                 Futures.transformAsync(future, result -> {
2338                     //Release label
2339                     if (result.isSuccessful()) {
2340                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2341                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2342                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2343                         return vpnService.removeVpnLabel(labelInput);
2344                     } else {
2345                         String errMsg =
2346                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2347                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2348                         LOG.error(errMsg);
2349                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2350                     }
2351                 }, MoreExecutors.directExecutor());
2352
2353             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2354
2355                 @Override
2356                 public void onFailure(@NonNull Throwable error) {
2357                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2358                         + "got external ip {}", label, extIp, error);
2359                 }
2360
2361                 @Override
2362                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2363                     if (result.isSuccessful()) {
2364                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2365                             + "from VPN {}", externalIp, externalVpn);
2366                     } else {
2367                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2368                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2369                     }
2370                 }
2371             }, MoreExecutors.directExecutor());
2372         } else {
2373             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2374                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2375         }
2376     }
2377
2378     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2379                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2380                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2381             throws ExecutionException, InterruptedException {
2382         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2383         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2384         if (routerName == null) {
2385             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2386             return;
2387         }
2388         //Get the external network provider type from networkId
2389         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2390         if (extNwProvType == null) {
2391             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2392                     networkUuid);
2393             return;
2394         }
2395         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2396          *  external network provided type is VxLAN
2397          */
2398         if (extNwProvType == ProviderTypes.VXLAN) {
2399             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2400             return;
2401         }
2402         //Get IPMaps from the DB for the router ID
2403         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2404         if (dbIpMaps.isEmpty()) {
2405             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2406             return;
2407         }
2408
2409         long tempLabel = NatConstants.INVALID_ID;
2410         for (IpMap dbIpMap : dbIpMaps) {
2411             String dbExternalIp = dbIpMap.getExternalIp();
2412             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2413             //Select the IPMap, whose external IP is the IP for which FIB is installed
2414             if (extIp.equals(dbExternalIp)) {
2415                 tempLabel = dbIpMap.getLabel();
2416                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2417                     tempLabel, dbExternalIp, routerId);
2418                 break;
2419             }
2420         }
2421         if (tempLabel == NatConstants.INVALID_ID) {
2422             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2423                     extIp, routerId);
2424             return;
2425         }
2426
2427         final long label = tempLabel;
2428         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2429         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2430                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2431                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2432         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2433
2434         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2435         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2436         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2437             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2438             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2439         }
2440         if (!switchOver) {
2441             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2442                     Futures.transformAsync(future, result -> {
2443                         //Release label
2444                         if (result.isSuccessful()) {
2445                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2446                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2447                             return vpnService.removeVpnLabel(labelInput);
2448                         } else {
2449                             String errMsg =
2450                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2451                                             + "prefix %s Failed - %s",
2452                                             dpnId, externalIp, result.getErrors());
2453                             LOG.error(errMsg);
2454                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2455                         }
2456                     }, MoreExecutors.directExecutor());
2457
2458             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2459
2460                 @Override
2461                 public void onFailure(@NonNull Throwable error) {
2462                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2463                 }
2464
2465                 @Override
2466                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2467                     if (result.isSuccessful()) {
2468                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2469                             + "from VPN {}", externalIp, vpnName);
2470                     } else {
2471                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2472                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2473                     }
2474                 }
2475             }, MoreExecutors.directExecutor());
2476         } else {
2477             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2478                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2479         }
2480     }
2481
2482     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2483             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2484             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2485         //Withdraw the corresponding routes from the BGP.
2486         //Get the network ID using the router ID.
2487         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2488                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2489         if (networkUuid == null) {
2490             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2491             return;
2492         }
2493
2494         if (externalIps == null || externalIps.isEmpty()) {
2495             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2496             return;
2497         }
2498
2499         if (vpnName == null) {
2500             //Get the VPN Name using the network ID
2501             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2502             if (vpnName == null) {
2503                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2504                     networkUuid, routerId);
2505                 return;
2506             }
2507         }
2508         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2509
2510         //Remove custom FIB routes
2511         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2512         for (String extIp : externalIps) {
2513             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2514                     writeFlowInvTx);
2515         }
2516     }
2517
2518     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2519         //Inform BGP about the route removal
2520         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2521         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2522         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2523     }
2524
2525     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2526             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2527         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2528         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2529             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2530         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2531     }
2532
2533     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2534             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2535         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2536         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2537         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2538         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2539     }
2540
2541     /**
2542      * router association to vpn.
2543      *
2544      * @param routerName - Name of router
2545      * @param routerId - router id
2546      * @param bgpVpnName BGP VPN name
2547      */
2548     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2549             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2550         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2551         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2552             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2553
2554             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2555
2556             if (bgpVpnId != NatConstants.INVALID_ID) {
2557                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2558                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2559                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2560                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2561                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2562                     getRoutersIdentifier(bgpVpnId), rtrs);
2563
2564                 // Get the allocated Primary NAPT Switch for this router
2565                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2566
2567                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2568                         routerId, bgpVpnId);
2569                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2570
2571                 // Get the group ID
2572                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2573                 createGroupId(getGroupIdKey(routerName));
2574                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2575                         extNwProvType);
2576             }
2577         }
2578     }
2579
2580     /**
2581      * router disassociation from vpn.
2582      *
2583      * @param routerName - Name of router
2584      * @param routerId - router id
2585      * @param bgpVpnName BGP VPN name
2586      */
2587     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2588             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2589         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2590         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2591             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2592             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2593
2594             // Get the allocated Primary NAPT Switch for this router
2595             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2596
2597             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2598             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2599
2600             // Get the group ID
2601             createGroupId(getGroupIdKey(routerName));
2602             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2603             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2604                     writeFlowInvTx, extNwProvType);
2605         }
2606     }
2607
2608     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2609         InstanceIdentifier<Routers> routerInstanceIndentifier =
2610             InstanceIdentifier.builder(ExtRouters.class)
2611                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2612         try {
2613             Optional<Routers> routerData = SingleTransactionDataBroker
2614                     .syncReadOptional(dataBroker,
2615                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2616             return routerData.isPresent() && routerData.get().isEnableSnat();
2617         } catch (ReadFailedException e) {
2618             LOG.error("Failed to read data for router id {}", routerUuid, e);
2619             return false;
2620         }
2621     }
2622
2623     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2624         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2625
2626         long changedVpnId = bgpVpnId;
2627         String idType = "BGP VPN";
2628         if (bgpVpnId == NatConstants.INVALID_ID) {
2629             changedVpnId = routerId;
2630             idType = "router";
2631         }
2632
2633         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2634         if (switches.isEmpty()) {
2635             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2636             return;
2637         }
2638         for (BigInteger dpnId : switches) {
2639             // Update the BGP VPN ID in the SNAT miss entry to group
2640             if (!dpnId.equals(primarySwitchId)) {
2641                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2642                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2643                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2644                 long groupId = createGroupId(getGroupIdKey(routerName));
2645                 if (!isSnatCfgd) {
2646                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2647                 }
2648
2649                 LOG.debug(
2650                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2651                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2652                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2653                 mdsalManager.addFlow(confTx, flowEntity);
2654             } else {
2655                 LOG.debug(
2656                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2657                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2658                 FlowEntity flowEntity =
2659                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2660                 mdsalManager.addFlow(confTx, flowEntity);
2661
2662                 LOG.debug(
2663                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2664                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2665                         idType, changedVpnId, primarySwitchId);
2666                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2667                         changedVpnId, confTx, extNwProvType);
2668
2669                 LOG.debug(
2670                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2671                                 + "which punts the packet to the controller in the Primary switch {}",
2672                         idType, changedVpnId, primarySwitchId);
2673                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2674
2675                 LOG.debug(
2676                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2677                                 + " outgoing packet to FIB Table in the Primary switch {}",
2678                         idType, changedVpnId, primarySwitchId);
2679                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2680
2681                 LOG.debug(
2682                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2683                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2684                                 + " {}", idType, changedVpnId, primarySwitchId);
2685                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2686
2687                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2688                         primarySwitchId);
2689                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2690                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2691                 if (vpnId != NatConstants.INVALID_ID) {
2692                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2693                 }
2694             }
2695         }
2696     }
2697
2698     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2699         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2700         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2701         if (ipPortMapping == null) {
2702             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2703             return;
2704         }
2705         // Get the External Gateway MAC Address
2706         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2707         if (extGwMacAddress != null) {
2708             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2709                     extGwMacAddress, routerId);
2710         } else {
2711             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2712                     routerId);
2713             return;
2714         }
2715         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2716             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2717                 String ipPortInternal = ipPortMap.getIpPortInternal();
2718                 String[] ipPortParts = ipPortInternal.split(":");
2719                 if (ipPortParts.length != 2) {
2720                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2721                     return;
2722                 }
2723                 String internalIp = ipPortParts[0];
2724                 String internalPort = ipPortParts[1];
2725                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2726                         internalIp, internalPort);
2727                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2728                 NAPTEntryEvent.Protocol protocol;
2729                 switch (protocolTypes) {
2730                     case TCP:
2731                         protocol = NAPTEntryEvent.Protocol.TCP;
2732                         break;
2733                     case UDP:
2734                         protocol = NAPTEntryEvent.Protocol.UDP;
2735                         break;
2736                     default:
2737                         protocol = NAPTEntryEvent.Protocol.TCP;
2738                 }
2739                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2740                 SessionAddress externalAddress =
2741                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2742                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2743                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2744                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2745                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2746                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2747             }
2748         }
2749     }
2750
2751     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2752                                                           long changedVpnId) {
2753
2754         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2755             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2756         List<MatchInfo> matches = new ArrayList<>();
2757         matches.add(MatchEthernetType.IPV4);
2758         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2759
2760         List<ActionInfo> actionsInfo = new ArrayList<>();
2761         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2762                 elanManager, idManager, changedVpnId, routerName);
2763         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2764         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2765                 actionsInfo);
2766         actionsInfo.add(new ActionGroup(groupId));
2767         List<InstructionInfo> instructions = new ArrayList<>();
2768         instructions.add(new InstructionApplyActions(actionsInfo));
2769         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2770         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2771             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2772             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2773
2774         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2775         return flowEntity;
2776     }
2777
2778     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2779                                                                         long changedVpnId) {
2780
2781         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2782                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2783         List<MatchInfo> matches = new ArrayList<>();
2784         matches.add(MatchEthernetType.IPV4);
2785         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2786
2787         List<InstructionInfo> instructions = new ArrayList<>();
2788         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2789
2790         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2791         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2792             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2793             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2794
2795         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2796         return flowEntity;
2797     }
2798
2799     // TODO : Replace this with ITM Rpc once its available with full functionality
2800     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2801         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2802
2803         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2804             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2805         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2806                 extNwProvType);
2807         mdsalManager.addFlow(confTx, flowEntity);
2808     }
2809
2810     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2811             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2812         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2813             dpId, routerName, changedVpnId);
2814         List<MatchInfo> matches = new ArrayList<>();
2815         matches.add(MatchEthernetType.IPV4);
2816
2817         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2818         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2819             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2820         }
2821         matches.add(new MatchTunnelId(tunnelId));
2822
2823         List<InstructionInfo> instructions = new ArrayList<>();
2824         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2825             MetaDataUtil.METADATA_MASK_VRFID));
2826         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2827         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2828         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2829         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2830             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2831             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2832         return flowEntity;
2833     }
2834
2835     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2836                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2837         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2838             dpnId, routerId, changedVpnId);
2839         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2840                 NwConstants.IP_PROT_TCP);
2841         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2842         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2843
2844         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2845                 NwConstants.IP_PROT_UDP);
2846         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2847         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2848
2849         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2850         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2851         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2852     }
2853
2854     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2855                                                            long changedVpnId, int protocol) {
2856         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2857             dpId, routerId, changedVpnId);
2858         BigInteger cookie = getCookieOutboundFlow(routerId);
2859         List<MatchInfo> matches = new ArrayList<>();
2860         matches.add(MatchEthernetType.IPV4);
2861         matches.add(new MatchIpProtocol((short)protocol));
2862         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2863
2864         List<InstructionInfo> instructions = new ArrayList<>();
2865         List<ActionInfo> actionsInfos = new ArrayList<>();
2866         actionsInfos.add(new ActionPuntToController());
2867         if (snatPuntTimeout != 0) {
2868             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2869         }
2870         instructions.add(new InstructionApplyActions(actionsInfos));
2871
2872         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2873         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2874             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2875         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2876         return flowEntity;
2877     }
2878
2879     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2880                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2881         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2882             dpnId, segmentId, changedVpnId);
2883         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2884         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2885     }
2886
2887     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2888
2889         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2890             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2891         List<MatchInfo> matches = new ArrayList<>();
2892         matches.add(MatchEthernetType.IPV4);
2893         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2894
2895         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2896         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2897         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2898         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2899         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2900
2901         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2902         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2903             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2904             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2905         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2906         return flowEntity;
2907     }
2908
2909     @Override
2910     protected ExternalRoutersListener getDataTreeChangeListener() {
2911         return ExternalRoutersListener.this;
2912     }
2913
2914     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2915                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2916         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2917                 routerName);
2918         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2919             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2920             if (subnetVpnId != -1) {
2921                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2922                     + "and vpnId {}", dpnId, subnetVpnId);
2923                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2924             }
2925         }
2926     }
2927 }