natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 145 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 146 import org.opendaylight.yangtools.yang.common.RpcResult;
 147 import org.opendaylight.yangtools.yang.common.Uint32;
 148 import org.opendaylight.yangtools.yang.common.Uint64;
 149 import org.slf4j.Logger;
 150 import org.slf4j.LoggerFactory;
 151
 152
 153 @Singleton
 154 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 155     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 156
 157     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 158     private static final Uint64 COOKIE_VM_LFIB_TABLE = Uint64.valueOf("8000022", 16).intern();
 159
 160     private final DataBroker dataBroker;
 161     private final ManagedNewTransactionRunner txRunner;
 162     private final IMdsalApiManager mdsalManager;
 163     private final ItmRpcService itmManager;
 164     private final OdlInterfaceRpcService odlInterfaceRpcService;
 165     private final IdManagerService idManager;
 166     private final NaptManager naptManager;
 167     private final NAPTSwitchSelector naptSwitchSelector;
 168     private final IBgpManager bgpManager;
 169     private final VpnRpcService vpnService;
 170     private final FibRpcService fibService;
 171     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 172     private final NaptEventHandler naptEventHandler;
 173     private final NaptPacketInHandler naptPacketInHandler;
 174     private final IFibManager fibManager;
 175     private final IVpnManager vpnManager;
 176     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 177     private final NatMode natMode;
 178     private final IElanService elanManager;
 179     private final JobCoordinator coordinator;
 180     private final IInterfaceManager interfaceManager;
 181     private final NatOverVxlanUtil natOverVxlanUtil;
 182     private final int snatPuntTimeout;
 183
 184     @Inject
 185     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 186                                    final ItmRpcService itmManager,
 187                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 188                                    final IdManagerService idManager,
 189                                    final NaptManager naptManager,
 190                                    final NAPTSwitchSelector naptSwitchSelector,
 191                                    final IBgpManager bgpManager,
 192                                    final VpnRpcService vpnService,
 193                                    final FibRpcService fibService,
 194                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 195                                    final NaptEventHandler naptEventHandler,
 196                                    final NaptPacketInHandler naptPacketInHandler,
 197                                    final IFibManager fibManager,
 198                                    final IVpnManager vpnManager,
 199                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 200                                    final NatserviceConfig config,
 201                                    final IElanService elanManager,
 202                                    final JobCoordinator coordinator,
 203                                    final NatOverVxlanUtil natOverVxlanUtil,
 204                                    final IInterfaceManager interfaceManager) {
 205         super(Routers.class, ExternalRoutersListener.class);
 206         this.dataBroker = dataBroker;
 207         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 208         this.mdsalManager = mdsalManager;
 209         this.itmManager = itmManager;
 210         this.odlInterfaceRpcService = odlInterfaceRpcService;
 211         this.idManager = idManager;
 212         this.naptManager = naptManager;
 213         this.naptSwitchSelector = naptSwitchSelector;
 214         this.bgpManager = bgpManager;
 215         this.vpnService = vpnService;
 216         this.fibService = fibService;
 217         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 218         this.naptEventHandler = naptEventHandler;
 219         this.naptPacketInHandler = naptPacketInHandler;
 220         this.fibManager = fibManager;
 221         this.vpnManager = vpnManager;
 222         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 223         this.elanManager = elanManager;
 224         this.coordinator = coordinator;
 225         this.interfaceManager = interfaceManager;
 226         this.natOverVxlanUtil = natOverVxlanUtil;
 227         if (config != null) {
 228             this.natMode = config.getNatMode();
 229             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 230         } else {
 231             this.natMode = NatMode.Controller;
 232             this.snatPuntTimeout = 0;
 233         }
 234     }
 235
 236     @Override
 237     @PostConstruct
 238     public void init() {
 239         LOG.info("{} init", getClass().getSimpleName());
 240         // This class handles ExternalRouters for Controller SNAT mode.
 241         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 242         if (natMode == NatMode.Controller) {
 243             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 244             NatUtil.createGroupIdPool(idManager);
 245         }
 246     }
 247
 248     @Override
 249     protected InstanceIdentifier<Routers> getWildCardPath() {
 250         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 251     }
 252
 253     @Override
 254     // TODO Clean up the exception handling
 255     @SuppressWarnings("checkstyle:IllegalCatch")
 256     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 257         // Populate the router-id-name container
 258         String routerName = routers.getRouterName();
 259         LOG.info("add : external router event for {}", routerName);
 260         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
 261         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 262         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 263         try {
 264             if (routers.isEnableSnat()) {
 265                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 266                     () -> Collections.singletonList(
 267                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 268                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 269                             Uint32 bgpVpnId = NatConstants.INVALID_ID;
 270                             if (bgpVpnUuid != null) {
 271                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 272                             }
 273                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 274                             // Allocate Primary Napt Switch for this router
 275                             Uint64 primarySwitchId = getPrimaryNaptSwitch(routerName);
 276                             if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 277                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 278                             }
 279                         }
 280                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 281             } else {
 282                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 283             }
 284         } catch (Exception ex) {
 285             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 286                     routerName, ex);
 287         }
 288     }
 289
 290     public void handleEnableSnat(Routers routers, Uint32 routerId, Uint64 primarySwitchId, Uint32 bgpVpnId,
 291                                  TypedWriteTransaction<Configuration> confTx) {
 292         String routerName = routers.getRouterName();
 293         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 294
 295         naptManager.initialiseExternalCounter(routers, routerId);
 296         subnetRegisterMapping(routers, routerId);
 297
 298         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 299             primarySwitchId, routerName);
 300
 301         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 302                 routers.getNetworkId());
 303         if (extNwProvType == null) {
 304             LOG.error("handleEnableSnat : External Network Provider Type missing");
 305             return;
 306         }
 307
 308         if (bgpVpnId != NatConstants.INVALID_ID) {
 309             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 310                     extNwProvType);
 311         } else {
 312             // write metadata and punt
 313             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 314             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, confTx);
 315             // Now install entries in SNAT tables to point to Primary for each router
 316             List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 317             for (Uint64 dpnId : switches) {
 318                 // Handle switches and NAPT switches separately
 319                 if (!dpnId.equals(primarySwitchId)) {
 320                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 321                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 322                 }
 323             }
 324         }
 325
 326         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 327         if (externalIps.isEmpty()) {
 328             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 329             return;
 330         } else {
 331             for (String externalIpAddrPrefix : externalIps) {
 332                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 333                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 334                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 335                 );
 336             }
 337         }
 338         LOG.debug("handleEnableSnat : Exit");
 339     }
 340
 341     private Uint64 getPrimaryNaptSwitch(String routerName) {
 342         // Allocate Primary Napt Switch for this router
 343         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 344         if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 345             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 346                 primarySwitchId, routerName);
 347             return primarySwitchId;
 348         }
 349         return selectNewNAPTSwitch(routerName);
 350     }
 351
 352     private Uint64 selectNewNAPTSwitch(String routerName) {
 353         // Allocated an id from VNI pool for the Router.
 354         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 355         Uint64 primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName, null);
 356         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 357
 358         return primarySwitchId;
 359     }
 360
 361     protected void installNaptPfibExternalOutputFlow(String routerName, Uint32 routerId, Uint64 dpnId,
 362                                                      TypedWriteTransaction<Configuration> confTx) {
 363         Uint32 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 364         if (extVpnId == NatConstants.INVALID_ID) {
 365             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 366             extVpnId = routerId;
 367         }
 368         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 369         if (externalIps.isEmpty()) {
 370             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 371                 dpnId, extVpnId);
 372             return;
 373         }
 374         for (String ip : externalIps) {
 375             Uuid subnetId = getSubnetIdForFixedIp(ip);
 376             if (subnetId != null) {
 377                 Uint32 subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 378                 if (subnetVpnId != NatConstants.INVALID_ID) {
 379                     extVpnId = subnetVpnId;
 380                 }
 381                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 382                     dpnId, extVpnId, subnetId);
 383                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 384                 if (postNaptFlowEntity != null) {
 385                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 386                 }
 387             }
 388         }
 389     }
 390
 391     @Nullable
 392     private Uuid getSubnetIdForFixedIp(String ip) {
 393         if (ip != null) {
 394             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 395             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 396             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 397         }
 398         LOG.error("getSubnetIdForFixedIp : ip is null");
 399         return null;
 400     }
 401
 402     protected void subnetRegisterMapping(Routers routerEntry, Uint32 segmentId) {
 403         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 404         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 405         int counter = 0;
 406         int extIpCounter = externalIps.size();
 407         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 408                 counter, extIpCounter);
 409         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 410         if (subnetIds == null) {
 411             return;
 412         }
 413         for (Uuid subnet : subnetIds) {
 414             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 415             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 416                 .builder(Subnetmaps.class)
 417                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 418                 .build();
 419             Optional<Subnetmap> sn;
 420             try {
 421                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 422                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 423             } catch (ReadFailedException e) {
 424                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 425                 sn = Optional.absent();
 426             }
 427             if (sn.isPresent()) {
 428                 // subnets
 429                 Subnetmap subnetmapEntry = sn.get();
 430                 String subnetString = subnetmapEntry.getSubnetIp();
 431                 String[] subnetSplit = subnetString.split("/");
 432                 String subnetIp = subnetSplit[0];
 433                 try {
 434                     InetAddress address = InetAddress.getByName(subnetIp);
 435                     if (address instanceof Inet6Address) {
 436                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 437                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 438                         continue;
 439                     }
 440                 } catch (UnknownHostException e) {
 441                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 442                     return;
 443                 }
 444                 String subnetPrefix = "0";
 445                 if (subnetSplit.length == 2) {
 446                     subnetPrefix = subnetSplit[1];
 447                 }
 448                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 449                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 450                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 451                 //externalIps
 452                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 453                         counter, extIpCounter);
 454                 if (extIpCounter != 0) {
 455                     if (counter < extIpCounter) {
 456                         String[] ipSplit = externalIps.get(counter).split("/");
 457                         String externalIp = ipSplit[0];
 458                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 459                         if (ipSplit.length == 2) {
 460                             extPrefix = ipSplit[1];
 461                         }
 462                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 463                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 464                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 465                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 466                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 467                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 468                     } else {
 469                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 470                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 471                         String[] ipSplit = externalIps.get(counter).split("/");
 472                         String externalIp = ipSplit[0];
 473                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 474                         if (ipSplit.length == 2) {
 475                             extPrefix = ipSplit[1];
 476                         }
 477                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 478                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 479                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 480                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 481                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 482                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 483                             externalIp, extPrefix);
 484                     }
 485                 }
 486                 counter++;
 487                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 488             } else {
 489                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 490             }
 491         }
 492     }
 493
 494     private void addOrDelDefFibRouteToSNAT(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 495             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 496             throws ExecutionException, InterruptedException {
 497         //Check if BGP VPN exists. If exists then invoke the new method.
 498         if (bgpVpnId != NatConstants.INVALID_ID) {
 499             if (bgpVpnUuid != null) {
 500                 String bgpVpnName = bgpVpnUuid.getValue();
 501                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 502                     bgpVpnId, bgpVpnName);
 503                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 504                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 505                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 506             }
 507             if (create) {
 508                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 509             } else {
 510                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 511             }
 512             return;
 513         }
 514
 515         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 516         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 517     }
 518
 519     private void addOrDelDefaultFibRouteForSNAT(String routerName, Uint32 routerId, boolean create,
 520             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 521         List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 522         if (switches.isEmpty()) {
 523             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 524             return;
 525         }
 526         if (routerId == NatConstants.INVALID_ID) {
 527             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 528                     + "default NAT route in FIB", routerName);
 529             return;
 530         }
 531         for (Uint64 dpnId : switches) {
 532             if (create) {
 533                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 534                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 535                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 536             } else {
 537                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 538                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 539                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 540             }
 541         }
 542     }
 543
 544     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId,
 545                                                      Uint32 bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 546         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 547         if (dpnIds.isEmpty()) {
 548             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 549                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 550             return;
 551         }
 552         for (Uint64 dpnId : dpnIds) {
 553             if (bgpVpnId != NatConstants.INVALID_ID) {
 554                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 555                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 556                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 557             } else {
 558                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 559                     + "in dpn {} for the internal vpn", routerId, dpnId);
 560                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 561             }
 562         }
 563     }
 564
 565     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 566                                                         TypedReadWriteTransaction<Configuration> confTx)
 567             throws ExecutionException, InterruptedException {
 568         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 569         if (dpnIds.isEmpty()) {
 570             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 571                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 572             return;
 573         }
 574         for (Uint64 dpnId : dpnIds) {
 575             if (bgpVpnId != NatConstants.INVALID_ID) {
 576                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 577                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 578                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 579             } else {
 580                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 581                     + "in dpn {} for the internal vpn", routerId, dpnId);
 582                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 583             }
 584         }
 585     }
 586
 587     protected void installOutboundMissEntry(String routerName, Uint32 routerId, Uint64 primarySwitchId,
 588         TypedWriteTransaction<Configuration> confTx) {
 589         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 590         if (routerId != NatConstants.INVALID_ID) {
 591             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 592                     primarySwitchId, routerId);
 593             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 594         } else {
 595             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 596                 + "createAndInstallMissEntry", routerName);
 597         }
 598     }
 599
 600     public String getFlowRefOutbound(Uint64 dpnId, short tableId, Uint32 routerID, int protocol) {
 601         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 602                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 603     }
 604
 605     private String getFlowRefNaptPreFib(Uint64 dpnId, short tableId, Uint32 vpnId) {
 606         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 607                 .FLOWID_SEPARATOR + vpnId;
 608     }
 609
 610     public Uint64 getCookieOutboundFlow(Uint32 routerId) {
 611         return Uint64.valueOf((NwConstants.COOKIE_OUTBOUND_NAPT_TABLE).toJava().add(
 612                 new BigInteger("0110001", 16)).add(BigInteger.valueOf(routerId.longValue())));
 613     }
 614
 615     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, Uint64 cookie) {
 616         long l4SrcPortField;
 617         long l4DstPortField;
 618         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 619
 620         if (protocol == NwConstants.IP_PROT_TCP) {
 621             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 622             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 623         } else {
 624             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 625             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 626         }
 627         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 628                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 629                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 630                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 631                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 632                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 633                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 634                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 635                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 636                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 637                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 638                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 639                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 640                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 641                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 642
 643         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 644                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 645     }
 646
 647     private FlowEntity buildIcmpDropFlow(Uint64 dpnId, Uint32 routerId, Uint32 vpnId) {
 648         List<MatchInfo> matches = new ArrayList<>();
 649         matches.add(MatchEthernetType.IPV4);
 650         matches.add(MatchIpProtocol.ICMP);
 651         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 652                 MetaDataUtil.METADATA_MASK_VRFID));
 653
 654         List<ActionInfo> actionInfos = new ArrayList<>();
 655         actionInfos.add(new ActionDrop());
 656
 657         List<InstructionInfo> instructions = new ArrayList<>();
 658         instructions.add(new InstructionApplyActions(actionInfos));
 659
 660         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 661                 NwConstants.IP_PROT_ICMP);
 662         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 663                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 664                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 665         return flow;
 666     }
 667
 668     protected FlowEntity buildOutboundFlowEntity(Uint64 dpId, Uint32 routerId, int protocol) {
 669         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 670         Uint64 cookie = getCookieOutboundFlow(routerId);
 671         List<MatchInfo> matches = new ArrayList<>();
 672         matches.add(MatchEthernetType.IPV4);
 673         matches.add(new MatchIpProtocol((short)protocol));
 674         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 675                 MetaDataUtil.METADATA_MASK_VRFID));
 676
 677         List<InstructionInfo> instructions = new ArrayList<>();
 678         List<ActionInfo> actionsInfos = new ArrayList<>();
 679         actionsInfos.add(new ActionPuntToController());
 680         if (snatPuntTimeout != 0) {
 681             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 682         }
 683         instructions.add(new InstructionApplyActions(actionsInfos));
 684
 685         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 686         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 687             5, flowRef, 0, 0,
 688             cookie, matches, instructions);
 689         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 690         return flowEntity;
 691     }
 692
 693     public void createOutboundTblEntry(Uint64 dpnId, Uint32 routerId, TypedWriteTransaction<Configuration> confTx) {
 694         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 695         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 696         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 697         mdsalManager.addFlow(confTx, tcpflowEntity);
 698
 699         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 700         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 701         mdsalManager.addFlow(confTx, udpflowEntity);
 702
 703         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 704         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 705         mdsalManager.addFlow(confTx, icmpDropFlow);
 706     }
 707
 708     @Nullable
 709     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 710         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 711         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 712         try {
 713             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 714                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 715                     .setSourceDpid(srcDpId)
 716                     .setDestinationDpid(dstDpId)
 717                     .setTunnelType(tunType)
 718                     .build());
 719             rpcResult = result.get();
 720             if (!rpcResult.isSuccessful()) {
 721                 tunType = TunnelTypeGre.class;
 722                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 723                     .setSourceDpid(srcDpId)
 724                     .setDestinationDpid(dstDpId)
 725                     .setTunnelType(tunType)
 726                     .build());
 727                 rpcResult = result.get();
 728                 if (!rpcResult.isSuccessful()) {
 729                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 730                             rpcResult.getErrors());
 731                 } else {
 732                     return rpcResult.getResult().getInterfaceName();
 733                 }
 734                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 735                         rpcResult.getErrors());
 736             } else {
 737                 return rpcResult.getResult().getInterfaceName();
 738             }
 739         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 740             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 741                     + "between {} and {}", srcDpId, dstDpId, e);
 742         }
 743
 744         return null;
 745     }
 746
 747     protected void installSnatMissEntryForPrimrySwch(Uint64 dpnId, String routerName, Uint32 routerId,
 748         TypedWriteTransaction<Configuration> confTx) {
 749
 750         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 751         // Install miss entry pointing to group
 752         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 753         mdsalManager.addFlow(confTx, flowEntity);
 754     }
 755
 756     protected void installSnatMissEntry(Uint64 dpnId, List<BucketInfo> bucketInfo,
 757             String routerName, Uint32 routerId) {
 758         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 759             dpnId, bucketInfo.get(0));
 760         // Install the select group
 761         Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 762             NatUtil.getGroupIdKey(routerName));
 763         if (groupId == NatConstants.INVALID_ID) {
 764             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 765             return;
 766         }
 767         GroupEntity groupEntity =
 768             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 769         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 770         mdsalManager.syncInstallGroup(groupEntity);
 771         // Install miss entry pointing to group
 772         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 773         if (flowEntity == null) {
 774             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 775                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 776                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 777             return;
 778         }
 779         mdsalManager.installFlow(flowEntity);
 780     }
 781
 782     void installGroup(Uint64 dpnId, String routerName, Uint32 groupId, List<BucketInfo> bucketInfo) {
 783         GroupEntity groupEntity =
 784             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 785         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 786         mdsalManager.syncInstallGroup(groupEntity);
 787     }
 788
 789     private FlowEntity buildSnatFlowEntity(Uint64 dpId, String routerName, Uint32 routerId, Uint32 groupId) {
 790         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 791                 dpId, routerName, groupId);
 792         List<MatchInfo> matches = new ArrayList<>();
 793         matches.add(MatchEthernetType.IPV4);
 794         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 795                 MetaDataUtil.METADATA_MASK_VRFID));
 796
 797         List<ActionInfo> actionsInfo = new ArrayList<>();
 798         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 799                 routerId, routerName);
 800         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
 801         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 802         actionsInfo.add(new ActionGroup(groupId.longValue()));
 803         List<InstructionInfo> instructions = new ArrayList<>();
 804         instructions.add(new InstructionApplyActions(actionsInfo));
 805         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 806         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 807                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 808                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 809
 810         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 811         return flowEntity;
 812     }
 813
 814     private FlowEntity buildSnatFlowEntityForPrmrySwtch(Uint64 dpId, String routerName, Uint32 routerId) {
 815
 816         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 817             routerName);
 818         List<MatchInfo> matches = new ArrayList<>();
 819         matches.add(MatchEthernetType.IPV4);
 820         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 821                 MetaDataUtil.METADATA_MASK_VRFID));
 822
 823         List<InstructionInfo> instructions = new ArrayList<>();
 824         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 825
 826         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 827         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 828             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 829             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 830
 831         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 832         return flowEntity;
 833     }
 834
 835     // TODO : Replace this with ITM Rpc once its available with full functionality
 836     protected void installTerminatingServiceTblEntry(Uint64 dpnId, String routerName, Uint32 routerId,
 837         TypedWriteTransaction<Configuration> confTx) {
 838
 839         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 840             dpnId, routerName);
 841         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 842         if (flowEntity == null) {
 843             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 844                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 845                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 846                     dpnId, routerName);
 847             return;
 848         }
 849         mdsalManager.addFlow(confTx, flowEntity);
 850
 851     }
 852
 853     private FlowEntity buildTsFlowEntity(Uint64 dpId, String routerName, Uint32 routerId) {
 854         List<MatchInfo> matches = new ArrayList<>();
 855         matches.add(MatchEthernetType.IPV4);
 856         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 857                 idManager, routerId, routerName);
 858         matches.add(new MatchTunnelId(tunnelId));
 859         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
 860         List<InstructionInfo> instructions = new ArrayList<>();
 861         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 862                 MetaDataUtil.METADATA_MASK_VRFID));
 863         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 864         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 865                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
 866                 NwConstants.COOKIE_TS_TABLE, matches, instructions);
 867         return flowEntity;
 868     }
 869
 870     public String getFlowRefTs(Uint64 dpnId, short tableId, Uint32 routerID) {
 871         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 872                 .FLOWID_SEPARATOR + routerID;
 873     }
 874
 875     public static String getFlowRefSnat(Uint64 dpnId, short tableId, String routerID) {
 876         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 877                 .FLOWID_SEPARATOR + routerID;
 878     }
 879
 880     protected void handleSwitches(Uint64 dpnId, String routerName, Uint32 routerId, Uint64 primarySwitchId) {
 881         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 882         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 883         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 884         List<BucketInfo> listBucketInfo = new ArrayList<>();
 885
 886         if (ifNamePrimary != null) {
 887             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 888             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 889                     interfaceManager, ifNamePrimary, routerId, true);
 890             if (listActionInfoPrimary.isEmpty()) {
 891                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 892                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 893                         ifNamePrimary);
 894             }
 895         } else {
 896             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 897                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 898         }
 899         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 900
 901         listBucketInfo.add(0, bucketPrimary);
 902         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 903     }
 904
 905     List<BucketInfo> getBucketInfoForNonNaptSwitches(Uint64 nonNaptSwitchId,
 906                                                      Uint64 primarySwitchId, String routerName, Uint32 routerId) {
 907         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 908         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 909         List<BucketInfo> listBucketInfo = new ArrayList<>();
 910
 911         if (ifNamePrimary != null) {
 912             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 913                     ifNamePrimary);
 914             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 915                     interfaceManager, ifNamePrimary, routerId, true);
 916             if (listActionInfoPrimary.isEmpty()) {
 917                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 918                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 919                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 920             }
 921         } else {
 922             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 923                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 924         }
 925         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 926
 927         listBucketInfo.add(0, bucketPrimary);
 928         return listBucketInfo;
 929     }
 930
 931     protected void handlePrimaryNaptSwitch(Uint64 dpnId, String routerName, Uint32 routerId,
 932         TypedWriteTransaction<Configuration> confTx) {
 933
 934        /*
 935         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 936         */
 937
 938         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 939
 940 /*
 941         List<BucketInfo> listBucketInfo = new ArrayList<>();
 942         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 943         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 944         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 945         listBucketInfo.add(0, bucketPrimary);
 946 */
 947
 948         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 949         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 950         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 951         installNaptPfibEntry(dpnId, routerId, confTx);
 952         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 953         if (networkId != null) {
 954             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 955             if (vpnUuid != null) {
 956                 Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 957                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 958                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 959                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 960                     if (vpnId != NatConstants.INVALID_ID) {
 961                         installNaptPfibEntry(dpnId, vpnId, null);
 962                     }
 963                     return Collections.emptyList();
 964                 });
 965             } else {
 966                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 967             }
 968         } else {
 969             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 970         }
 971     }
 972
 973     public void installNaptPfibEntry(Uint64 dpnId, Uint32 segmentId,
 974             @Nullable TypedWriteTransaction<Configuration> confTx) {
 975         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 976         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 977         if (confTx != null) {
 978             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 979         } else {
 980             mdsalManager.installFlow(naptPfibFlowEntity);
 981         }
 982     }
 983
 984     public FlowEntity buildNaptPfibFlowEntity(Uint64 dpId, Uint32 segmentId) {
 985
 986         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 987         List<MatchInfo> matches = new ArrayList<>();
 988         matches.add(MatchEthernetType.IPV4);
 989         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId.longValue()),
 990                 MetaDataUtil.METADATA_MASK_VRFID));
 991
 992         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 993         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 994         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
 995         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 996         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 997
 998         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
 999         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1000             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1001             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1002         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1003         return flowEntity;
1004     }
1005
1006     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, Uint64 dpnId, Routers router,
1007                                          Uint32 routerId, String routerName, String externalIp) {
1008         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1009             dpnId, routerId, externalIp);
1010         Uuid networkId = router.getNetworkId();
1011         if (networkId == null) {
1012             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1013             return;
1014         }
1015         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1016         if (vpnName == null) {
1017             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1018                     + "configuration {} in router {}", networkId, externalIp, routerId);
1019             return;
1020         }
1021         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1022             externalIp, networkId, router, confTx);
1023         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1024             dpnId, routerId, externalIp);
1025     }
1026
1027     public void advToBgpAndInstallFibAndTsFlows(final Uint64 dpnId, final short tableId, final String vpnName,
1028                                                 final Uint32 routerId, final String routerName, final String externalIp,
1029                                                 final Uuid extNetworkId, @Nullable final Routers router,
1030                                                 final TypedWriteTransaction<Configuration> confTx) {
1031         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1032                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1033         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1034         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1035         if (rd == null || rd.isEmpty()) {
1036             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1037             return;
1038         }
1039         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1040         if (extNwProvType == null) {
1041             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1042             return;
1043         }
1044         if (extNwProvType == ProviderTypes.VXLAN) {
1045             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1046                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1047             return;
1048         }
1049         //Generate VPN label for the external IP
1050         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1051             .setIpPrefix(externalIp).build();
1052         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1053
1054         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1055         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1056             if (result.isSuccessful()) {
1057                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1058                 GenerateVpnLabelOutput output = result.getResult();
1059                 final Uint32 label = output.getLabel();
1060
1061                 int externalIpInDsFlag = 0;
1062                 //Get IPMaps from the DB for the router ID
1063                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1064                 for (IpMap dbIpMap : dbIpMaps) {
1065                     String dbExternalIp = dbIpMap.getExternalIp();
1066                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1067                     if (dbExternalIp.contains(externalIp)) {
1068                         String dbInternalIp = dbIpMap.getInternalIp();
1069                         IpMapKey dbIpMapKey = dbIpMap.key();
1070                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1071                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1072                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1073                             .setExternalIp(dbExternalIp).setLabel(label).build();
1074                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1075                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1076                         externalIpInDsFlag++;
1077                     }
1078                 }
1079                 if (externalIpInDsFlag <= 0) {
1080                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1081                             + "Failed to update label {} for routerId {} in DS",
1082                             externalIp, label, routerId);
1083                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1084                         + " found in DS for router %s", label, externalIp, routerId);
1085                     return Futures.immediateFailedFuture(new Exception(errMsg));
1086                 }
1087                 //Inform BGP
1088                 Uint32 l3vni = Uint32.ZERO;
1089                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1090                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
1091                 }
1092                 Routers extRouter = router != null ? router :
1093                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1094                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1095                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1096                     RouteOrigin.STATIC, dpnId);
1097
1098                 //Install custom FIB routes
1099                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1100                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1101                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1102                         extNwProvType);
1103                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1104
1105                 //Install custom FIB routes - FIB table.
1106                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1107                         routerName, externalIp);
1108                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1109                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1110                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1111                 }
1112                 String externalVpn = vpnName;
1113                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1114                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1115                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1116                             externalSubnetId);
1117                     if (externalSubnet.isPresent()) {
1118                         externalVpn =  externalSubnetId.getValue();
1119                     }
1120                 }
1121                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1122                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1123                     .setVpnName(externalVpn)
1124                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1125                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1126                     .setInstruction(fibTableCustomInstructions).build();
1127                 return fibService.createFibEntry(input);
1128             } else {
1129                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1130                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1131                     externalIp, vpnName, result.getErrors());
1132                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1133             }
1134         }, MoreExecutors.directExecutor());
1135
1136         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1137
1138             @Override
1139             public void onFailure(@NonNull Throwable error) {
1140                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1141             }
1142
1143             @Override
1144             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1145                 if (result.isSuccessful()) {
1146                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1147                             externalIp);
1148                 } else {
1149                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1150                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1151                 }
1152             }
1153         }, MoreExecutors.directExecutor());
1154     }
1155
1156     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1157             String externalIp) {
1158         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1159         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1160         Uint32 externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1161                 externalIp, router);
1162         int instructionIndex = 0;
1163         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1164             Uint64 subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId.longValue());
1165             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1166                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1167             instructionIndex++;
1168         }
1169
1170         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1171         return fibTableCustomInstructions;
1172     }
1173
1174     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, short tableId,
1175         TypedWriteTransaction<Configuration> confTx) {
1176         List<MatchInfo> matches = new ArrayList<>();
1177         matches.add(MatchEthernetType.MPLS_UNICAST);
1178         matches.add(new MatchMplsLabel(serviceId.longValue()));
1179
1180         List<Instruction> instructions = new ArrayList<>();
1181         List<ActionInfo> actionsInfos = new ArrayList<>();
1182         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1183         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1184         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1185         instructions.add(writeInstruction);
1186         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1187
1188         // Install the flow entry in L3_LFIB_TABLE
1189         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1190
1191         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1192             10, flowRef, 0, 0,
1193             COOKIE_VM_LFIB_TABLE, matches, instructions);
1194
1195         mdsalManager.addFlow(confTx, dpId, flowEntity);
1196
1197         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1198     }
1199
1200     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId, Uint32 l3Vni,
1201         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1202         ProviderTypes extNwProvType) {
1203
1204         List<MatchInfo> mkMatches = new ArrayList<>();
1205
1206         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1207             dpnId, serviceId, customInstructions);
1208
1209         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1210             mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
1211         } else {
1212             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
1213         }
1214
1215         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1216             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1217                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1218                 String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
1219                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(serviceId.longValue()))),
1220                 mkMatches, customInstructions);
1221
1222         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1223     }
1224
1225     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(Uint32 routerId) {
1226         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1227             new RouterIdsKey(routerId)).build();
1228     }
1229
1230     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
1231         return NatConstants.SNAT_FLOWID_PREFIX + dpnId.toString() + NwConstants.FLOWID_SEPARATOR + tableId
1232                 + NwConstants.FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1233     }
1234
1235     @Override
1236     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1237         LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1238         String routerName = original.getRouterName();
1239         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1240         if (routerId == NatConstants.INVALID_ID) {
1241             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1242             return;
1243         }
1244         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1245             List<ListenableFuture<Void>> futures = new ArrayList<>();
1246             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1247                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1248                     Uint32 bgpVpnId = NatConstants.INVALID_ID;
1249                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1250                     if (bgpVpnUuid != null) {
1251                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1252                     }
1253                     //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1254                     /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1255                      * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1256                      */
1257                     Uint64 dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1258                     boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(Uint64
1259                             .valueOf(BigInteger.ZERO)));
1260                     Uuid networkId = original.getNetworkId();
1261                     // Check if its update on SNAT flag
1262                     boolean originalSNATEnabled = original.isEnableSnat();
1263                     boolean updatedSNATEnabled = update.isEnableSnat();
1264                     LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1265                             + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1266                         originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1267                     // Cluster Reboot Case Handling
1268                     // 1. DPN not elected during add event(due to none of the OVS connected)
1269                     // 2. Update event called with changes of parameters(but enableSnat is not changed)
1270                     // 3. First Elect dpnId and process other changes with valid dpnId
1271                     if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1272                         if (originalSNATEnabled && !updatedSNATEnabled) {
1273                             if (isPrimaryNaptSwitchNotSelected) {
1274                                 LOG.info("No Action to be taken when SNAT is disabled "
1275                                     + "with no Napt Switch Election for Router {}", routerName);
1276                                 return;
1277                             }
1278                             //SNAT disabled for the router
1279                             Uuid networkUuid = original.getNetworkId();
1280                             LOG.info("update : SNAT disabled for Router {}", routerName);
1281                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1282                             handleDisableSnat(original, networkUuid, externalIps, false, null,
1283                                     dpnId, routerId, removeFlowInvTx);
1284                         }  else if (updatedSNATEnabled) {
1285                             LOG.info("update : SNAT enabled for Router {}", routerName);
1286                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1287                                 true, writeFlowInvTx);
1288                             if (isPrimaryNaptSwitchNotSelected) {
1289                                 dpnId = selectNewNAPTSwitch(routerName);
1290                                 if (dpnId != null && !dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1291                                     handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1292                                 } else {
1293                                     LOG.error("update : Failed to elect Napt Switch During update event"
1294                                         + " of router {}", routerName);
1295                                 }
1296                             }
1297                         }
1298                         LOG.info("update : no need to process external/subnet changes as it's will taken care"
1299                             + "in handleDisableSnat/handleEnableSnat");
1300                         return;
1301                     }
1302                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1303                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1304                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1305                     }
1306
1307                     if (updatedSNATEnabled != originalSNATEnabled) {
1308                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1309                                 + "handleDisableSnat/handleEnableSnat");
1310                         return;
1311                     }
1312                     //Check if the Update is on External IPs
1313                     LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1314                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1315                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1316
1317                     //Check if the External IPs are removed during the update.
1318                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1319                     removedExternalIps.removeAll(updatedExternalIps);
1320                     if (removedExternalIps.size() > 0) {
1321                         LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1322                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1323                                 removedExternalIps, original.getExtGwMacAddress(),
1324                                 dpnId, networkId);
1325
1326                         for (String removedExternalIp : removedExternalIps) {
1327                 /*
1328                     1) Remove the mappings in the IntExt IP model which has external IP.
1329                     2) Remove the external IP in the ExternalCounter model.
1330                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1331                        least loaded external IP.
1332                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1333                     4) Increase the count of the allocated external IP by one.
1334                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1335                      i.e. the route for the external IP is absent.
1336                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1337                      the removed external IPs and also from the model.
1338                     7) Advertise to the BGP for removing the route for the removed external IPs.
1339                  */
1340
1341                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1342                             String externalIp = externalIpParts[0];
1343                             String externalIpPrefix = externalIpParts[1];
1344                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1345
1346                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1347                                     + "entries for removed external IP {}", externalIpAddrStr);
1348                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1349                             String vpnName = "";
1350                             if (vpnUuId != null) {
1351                                 vpnName = vpnUuId.getValue();
1352                             }
1353                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1354                                     update.getExtGwMacAddress(), removeFlowInvTx);
1355
1356                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1357                             //Get the internal IPs which are associated to the removed external IPs
1358                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1359                             List<String> removedInternalIps = new ArrayList<>();
1360                             for (IpMap ipMap : ipMaps) {
1361                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1362                                     removedInternalIps.add(ipMap.getInternalIp());
1363                                 }
1364                             }
1365
1366                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1367                             for (String removedInternalIp : removedInternalIps) {
1368                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1369                                                 + "router ID {} from the IntExtIP model",
1370                                         removedInternalIp, routerId);
1371                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1372                             }
1373
1374                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1375                                             + "router ID {} from the ExternalIpsCounter model.",
1376                                     externalIpAddrStr, routerId);
1377                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1378
1379                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1380                                     + "whose external IPs were removed.");
1381                             for (String removedInternalIp : removedInternalIps) {
1382                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1383                                         removedInternalIp, writeFlowInvTx);
1384                             }
1385                             LOG.debug("update : Remove the NAPT translation entries from "
1386                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1387                             //Get the internalIP and internal Port which were associated to the removed external IP.
1388                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1389                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1390                                     .builder(IntextIpPortMap.class)
1391                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1392                             Optional<IpPortMapping> ipPortMapping;
1393                             try {
1394                                 ipPortMapping = SingleTransactionDataBroker
1395                                             .syncReadOptional(dataBroker,
1396                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1397                             } catch (ReadFailedException e) {
1398                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1399                                 ipPortMapping = Optional.absent();
1400                             }
1401
1402                             if (ipPortMapping.isPresent()) {
1403                                 for (IntextIpProtocolType intextIpProtocolType :
1404                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1405                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1406                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1407                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1408                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1409                                             List<String> removedInternalIpPorts =
1410                                                     protoTypesIntIpPortsMap.get(protoType);
1411                                             if (removedInternalIpPorts != null) {
1412                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1413                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1414                                             } else {
1415                                                 removedInternalIpPorts = new ArrayList<>();
1416                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1417                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1418                                             }
1419                                         }
1420                                     }
1421                                 }
1422                             }
1423
1424                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1425                             // the removed external IP.
1426                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1427                                     protoTypesIntIpPortsMap.entrySet();
1428                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1429                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1430                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1431                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1432                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1433                                     // Remove the IP port map from the intext-ip-port-map model,
1434                                     // which were containing the removed external IP
1435                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1436                                             protocolType);
1437                                     //Remove the IP port incomint packer map.
1438                                     naptPacketInHandler.removeIncomingPacketMap(
1439                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1440                                     String[] removedInternalIpPortParts = removedInternalIpPort
1441                                             .split(NatConstants.COLON_SEPARATOR);
1442                                     if (removedInternalIpPortParts.length == 2) {
1443                                         String removedInternalIp = removedInternalIpPortParts[0];
1444                                         String removedInternalPort = removedInternalIpPortParts[1];
1445                                         List<String> removedInternalPortsList =
1446                                                 internalIpPortMap.get(removedInternalPort);
1447                                         if (removedInternalPortsList != null) {
1448                                             removedInternalPortsList.add(removedInternalPort);
1449                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1450                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1451                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1452                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1453                                             //Remove the NAPT translation entries from Outbound NAPT table
1454                                             naptEventHandler.removeNatFlows(dpnId,
1455                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1456                                                 routerId, removedInternalIp,
1457                                                 Integer.parseInt(removedInternalPort),
1458                                                 protocolType.getName());
1459                                             naptEventHandler.removeNatFlows(dpnId,
1460                                                 NwConstants.INBOUND_NAPT_TABLE,
1461                                                 routerId, removedInternalIp,
1462                                                 Integer.parseInt(removedInternalPort),
1463                                                 protocolType.getName());
1464                                         } else {
1465                                             removedInternalPortsList = new ArrayList<>();
1466                                             removedInternalPortsList.add(removedInternalPort);
1467                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1468                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1469                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1470                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1471                                             //Remove the NAPT translation entries from Outbound NAPT table
1472                                             naptEventHandler.removeNatFlows(dpnId,
1473                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1474                                                 routerId, removedInternalIp,
1475                                                 Integer.parseInt(removedInternalPort),
1476                                                 protocolType.getName());
1477                                             naptEventHandler.removeNatFlows(dpnId,
1478                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1479                                                 Integer.parseInt(removedInternalPort),
1480                                                 protocolType.getName());
1481                                         }
1482                                     }
1483                                 }
1484                             }
1485
1486                             // Delete the entry from SnatIntIpPortMap DS
1487                             Set<String> internalIps = internalIpPortMap.keySet();
1488                             for (String internalIp : internalIps) {
1489                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1490                                         + "model SnatIntIpPortMap", internalIp);
1491                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1492                             }
1493
1494                             naptManager.removeNaptPortPool(externalIp);
1495                         }
1496                         LOG.debug(
1497                                 "update : End processing of the External IPs removal for router {}", routerName);
1498                     }
1499
1500                     //Check if the External IPs are added during the update.
1501                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1502                     addedExternalIps.removeAll(originalExternalIps);
1503                     if (addedExternalIps.size() != 0) {
1504                         LOG.debug("update : Start processing of the External IPs addition for router {}",
1505                             routerName);
1506                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1507                                 update.getExtGwMacAddress(), dpnId,
1508                                 update.getNetworkId());
1509
1510                         for (String addedExternalIp : addedExternalIps) {
1511                 /*
1512                     1) Do nothing in the IntExtIp model.
1513                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1514                  */
1515                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1516                             String externalIp = externalIpParts[0];
1517                             String externalIpPrefix = externalIpParts[1];
1518                             String externalpStr = externalIp + "/" + externalIpPrefix;
1519                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1520                                             + "router ID {} in the ExternalIpsCounter model.",
1521                                     externalpStr, routerId);
1522                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1523                             subnetRegisterMapping(update, routerId);
1524                             LOG.info("update : Installing fib flow fo newly added Ips");
1525                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1526                         }
1527                         LOG.debug(
1528                                 "update : End processing of the External IPs addition during the update operation");
1529                     }
1530
1531                     //Check if its Update on subnets
1532                     LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1533                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1534                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1535                     Set<Uuid> addedSubnetIds =
1536                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1537                     if (originalSubnetIds != null) {
1538                         addedSubnetIds.removeAll(originalSubnetIds);
1539                     }
1540
1541                     //Check if the Subnet IDs are added during the update.
1542                     if (addedSubnetIds.size() != 0) {
1543                         LOG.debug(
1544                                 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1545                         for (Uuid addedSubnetId : addedSubnetIds) {
1546                 /*
1547                     1) Select the least loaded external IP for the subnet and store the mapping of the
1548                     subnet IP and the external IP in the IntExtIp model.
1549                     2) Increase the count of the selected external IP by one.
1550                     3) Advertise to the BGP if external IP is allocated for the first time for the
1551                     router i.e. the route for the external IP is absent.
1552                  */
1553                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1554                             if (subnetIp != null) {
1555                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1556                                         writeFlowInvTx);
1557                             }
1558                         }
1559                         LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1560                     }
1561
1562                     //Check if the Subnet IDs are removed during the update.
1563                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1564                     removedSubnetIds.removeAll(updatedSubnetIds);
1565                     if (removedSubnetIds.size() != 0) {
1566                         LOG.debug(
1567                                 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1568                         for (Uuid removedSubnetId : removedSubnetIds) {
1569                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1570                             if (subnetAddr != null) {
1571                     /*
1572                         1) Remove the subnet IP and the external IP in the IntExtIp map
1573                         2) Decrease the count of the coresponding external IP by one.
1574                         3) Advertise to the BGP for removing the routes of the corresponding external
1575                         IP if its not allocated to any other internal IP.
1576                     */
1577
1578                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1579                                         subnetAddr[0] + "/" + subnetAddr[1]);
1580                                 if (externalIp == null) {
1581                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1582                                             routerId, subnetAddr[0]);
1583                                     return;
1584                                 }
1585
1586                                 naptManager.updateCounter(routerId, externalIp, false);
1587                                 // Traverse entire model of external-ip counter whether external ip is not
1588                                 // used by any other internal ip in any router
1589                                 if (!isExternalIpAllocated(externalIp)) {
1590                                     LOG.debug("update : external ip is not allocated to any other "
1591                                             + "internal IP so proceeding to remove routes");
1592                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1593                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1594                                             removeFlowInvTx);
1595                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1596                                                     + "router {} with networkId {} and externalIp {}",
1597                                             dpnId, routerId, networkId, externalIp);
1598                                 }
1599
1600                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1601                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1602                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1603                             }
1604                         }
1605                         LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1606                     }
1607                 }));
1608             }));
1609             return futures;
1610         }, NatConstants.NAT_DJC_MAX_RETRIES);
1611     }
1612
1613     private boolean isExternalIpAllocated(String externalIp) {
1614         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1615         Optional<ExternalIpsCounter> externalCountersData;
1616         try {
1617             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1618                         LogicalDatastoreType.OPERATIONAL, id);
1619         } catch (ReadFailedException e) {
1620             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1621             externalCountersData = Optional.absent();
1622         }
1623         if (externalCountersData.isPresent()) {
1624             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1625             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1626                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1627                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1628                         if (externalIpCount.getCounter().toJava() != 0) {
1629                             return true;
1630                         }
1631                         break;
1632                     }
1633                 }
1634             }
1635         }
1636         return false;
1637     }
1638
1639     private void allocateExternalIp(Uint64 dpnId, Routers router, Uint32 routerId, String routerName,
1640             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1641         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1642         try {
1643             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1644             if (address instanceof Inet6Address) {
1645                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1646                 return;
1647             }
1648         } catch (UnknownHostException e) {
1649             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1650             return;
1651         }
1652         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1653         if (leastLoadedExtIpAddr != null) {
1654             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1655             String leastLoadedExtIp = externalIpParts[0];
1656             String leastLoadedExtIpPrefix = externalIpParts[1];
1657             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1658             subnetIp = subnetIpParts[0];
1659             String subnetIpPrefix = subnetIpParts[1];
1660             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1661             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1662                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1663                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1664             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1665
1666
1667             // Check if external IP is already assigned a route. (i.e. External IP is previously
1668             // allocated to any of the subnets)
1669             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1670             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1671             Uint32 label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1672             if (label != null) {
1673                 //update
1674                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1675                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1676                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1677                     label, internalIp, leastLoadedExtIpAddrStr);
1678                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1679                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1680                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1681                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1682                 return;
1683             }
1684
1685             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1686             // for the first time and hence not having a route.
1687             //Get the VPN Name using the network ID
1688             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1689             if (vpnName != null) {
1690                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1691                 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1692                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1693                         + "added after gateway-set");
1694                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1695                     if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1696                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1697                         return;
1698                     }
1699                 }
1700                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1701                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1702                     writeFlowInvTx);
1703             }
1704         }
1705     }
1706
1707     @Nullable
1708     protected Uint32 checkExternalIpLabel(Uint32 routerId, String externalIp) {
1709         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1710         for (IpMap ipMap : ipMaps) {
1711             if (ipMap.getExternalIp().equals(externalIp)) {
1712                 if (ipMap.getLabel() != null) {
1713                     return ipMap.getLabel();
1714                 }
1715             }
1716         }
1717         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1718         return null;
1719     }
1720
1721     @Override
1722     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1723         LOG.trace("remove : Router delete method");
1724         /*
1725         ROUTER DELETE SCENARIO
1726         1) Get the router ID from the event.
1727         2) Build the cookie information from the router ID.
1728         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1729         4) Build the flow with the cookie value.
1730         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1731         6) Remove the flows from the other switches which points to the primary and secondary
1732          switches for the flows related the router ID.
1733         7) Get the list of external IP address maintained for the router ID.
1734         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1735         9) Withdraw the corresponding routes from the BGP.
1736          */
1737
1738         if (identifier == null || router == null) {
1739             LOG.error("remove : returning without processing since routers is null");
1740             return;
1741         }
1742
1743         String routerName = router.getRouterName();
1744         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1745             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1746                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1747                         routerName);
1748                 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1749                 if (routerId == NatConstants.INVALID_ID) {
1750                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1751                             routerName);
1752                     return;
1753                 }
1754                 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1755                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1756                 if (bgpVpnUuid != null) {
1757                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1758                 }
1759                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1760                         tx);
1761                 Uuid networkUuid = router.getNetworkId();
1762
1763                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1764                 if (primarySwitchId == null || primarySwitchId.equals(Uint64.ZERO)) {
1765                     // No NAPT switch for external router, probably because the router is not attached to
1766                     // any
1767                     // internal networks
1768                     LOG.debug(
1769                             "No NAPT switch for router {}, check if router is attached to any internal "
1770                                     + "network",
1771                             routerName);
1772                     return;
1773                 } else {
1774                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1775                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1776                             routerId, tx);
1777                 }
1778                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1779                     == NatConstants.INVALID_ID) {
1780                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1781                 }
1782             })), NatConstants.NAT_DJC_MAX_RETRIES);
1783     }
1784
1785     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1786                                   boolean routerFlag, @Nullable String vpnName, Uint64 naptSwitchDpnId,
1787                                   Uint32 routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1788         LOG.info("handleDisableSnat : Entry");
1789         String routerName = router.getRouterName();
1790         try {
1791             if (routerFlag) {
1792                 removeNaptSwitch(routerName);
1793             } else {
1794                 updateNaptSwitch(routerName, Uint64.valueOf(BigInteger.ZERO));
1795             }
1796
1797             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1798             naptManager.removeExternalCounter(routerId);
1799
1800             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1801             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1802                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1803                     + "router ID {} from RouterNaptSwitch model", routerId);
1804                 return;
1805             }
1806             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1807                     networkUuid);
1808             if (extNwProvType == null) {
1809                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1810                 return;
1811             }
1812             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1813             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1814                     externalSubnetList, removeFlowInvTx, extNwProvType);
1815             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1816             String externalSubnetVpn = null;
1817             for (Uuid externalSubnetId : externalSubnetList) {
1818                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1819                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1820                 if (externalSubnet.isPresent()) {
1821                     externalSubnetVpn =  externalSubnetId.getValue();
1822                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1823                             router.getExtGwMacAddress(), removeFlowInvTx);
1824                 }
1825             }
1826             if (externalSubnetVpn == null) {
1827                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1828                         router.getExtGwMacAddress(), removeFlowInvTx);
1829             }
1830             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1831             // for the router ID.
1832             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1833                     + "router ID {} in the DS", routerId);
1834             naptManager.removeMapping(routerId);
1835         } catch (InterruptedException | ExecutionException e) {
1836             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1837         }
1838         LOG.info("handleDisableSnat : Exit");
1839     }
1840
1841     // TODO Clean up the exception handling
1842     @SuppressWarnings("checkstyle:IllegalCatch")
1843     public void handleDisableSnatInternetVpn(String routerName, Uint32 routerId, Uuid networkUuid,
1844                                              @NonNull Collection<String> externalIps,
1845                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1846         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1847                 + "with internet vpn {}", routerName, vpnId);
1848         try {
1849             Uint64 naptSwitchDpnId = null;
1850             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1851                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1852             Optional<RouterToNaptSwitch> rtrToNapt;
1853             try {
1854                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1855                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1856             } catch (ReadFailedException e) {
1857                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1858                 rtrToNapt = Optional.absent();
1859             }
1860             if (rtrToNapt.isPresent()) {
1861                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1862             }
1863             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1864
1865             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1866                     writeFlowInvTx);
1867             try {
1868                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1869                 if (extGwMacAddress != null) {
1870                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1871                             + "External Router ID {}", extGwMacAddress, routerId);
1872                 } else {
1873                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1874                             + "External Router ID {}", routerId);
1875                     return;
1876                 }
1877                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1878                         writeFlowInvTx);
1879             } catch (Exception ex) {
1880                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1881                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1882             }
1883             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1884                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1885             }
1886         } catch (InterruptedException | ExecutionException e) {
1887             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1888                     + "with internet vpn {}", routerName, vpnId, e);
1889         }
1890         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1891                 routerName, vpnId);
1892     }
1893
1894     // TODO Clean up the exception handling
1895     @SuppressWarnings("checkstyle:IllegalCatch")
1896     public void updateNaptSwitch(String routerName, Uint64 naptSwitchId) {
1897         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1898             .setPrimarySwitchId(naptSwitchId).build();
1899         try {
1900             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1901                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1902         } catch (Exception ex) {
1903             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1904                 naptSwitchId, routerName);
1905         }
1906         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1907             naptSwitchId, routerName);
1908     }
1909
1910     protected void removeNaptSwitch(String routerName) {
1911         // Remove router and switch from model
1912         InstanceIdentifier<RouterToNaptSwitch> id =
1913             InstanceIdentifier.builder(NaptSwitches.class)
1914                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1915         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1916         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1917         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1918         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1919     }
1920
1921     public void removeNaptFlowsFromActiveSwitch(Uint32 routerId, String routerName,
1922                                                 Uint64 dpnId, Uuid networkId, String vpnName,
1923                                                 @NonNull Collection<String> externalIps,
1924                                                 Collection<Uuid> externalSubnetList,
1925                                                 TypedReadWriteTransaction<Configuration> confTx,
1926                                                 ProviderTypes extNwProvType)
1927             throws InterruptedException, ExecutionException {
1928
1929         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1930         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1931
1932         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1933         // traffic which comes from the  VMs of the NAPT switches)
1934         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1935         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1936
1937         LOG.info(
1938             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1939                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1940         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1941
1942         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1943         // traffic which comes from the VMs of the non NAPT switches)
1944         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1945                 elanManager, idManager, routerId, routerName);
1946         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
1947         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1948         LOG.info(
1949             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1950                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1951         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1952
1953         //Remove the flow table 25->44 from NAPT Switch
1954         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1955             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1956         }
1957
1958         //Remove the Outbound flow entry which forwards the packet to FIB Table
1959         LOG.info(
1960             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1961                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1962
1963         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1964             NwConstants.IP_PROT_TCP);
1965         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1966             outboundTcpNatFlowRef);
1967         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1968
1969         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1970             NwConstants.IP_PROT_UDP);
1971         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1972             outboundUdpNatFlowRef);
1973         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1974
1975         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1976             NwConstants.IP_PROT_ICMP);
1977         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1978             icmpDropFlowRef);
1979         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1980         boolean lastRouterOnExternalNetwork =
1981             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1982         if (lastRouterOnExternalNetwork) {
1983             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1984         }
1985         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1986         // External Subnet Vpn Id.
1987         for (Uuid externalSubnetId : externalSubnetList) {
1988             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1989             if (subnetVpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1990                 dataBroker, externalSubnetId, routerName, dpnId)) {
1991                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1992                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1993                     natPfibSubnetFlowRef);
1994                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1995                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1996                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1997                     subnetVpnId, dpnId);
1998             }
1999         }
2000
2001         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
2002         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
2003         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
2004
2005         LOG.info(
2006             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2007             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2008         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2009
2010         if (lastRouterOnExternalNetwork) {
2011             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2012             // - This does not work since ext-routers is deleted already - no network info
2013             //Get the VPN ID from the ExternalNetworks model
2014             Uint32 vpnId = NatConstants.INVALID_ID;
2015             if (vpnName == null || vpnName.isEmpty()) {
2016                 // ie called from router delete cases
2017                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2018                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2019                 if (vpnUuid != null) {
2020                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2021                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2022                               + "or disableSNAT scenario", vpnId, networkId);
2023                 }
2024             } else {
2025                 // ie called from disassociate vpn case
2026                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2027                     vpnName);
2028                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2029                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2030                     vpnId);
2031             }
2032
2033             if (vpnId != NatConstants.INVALID_ID) {
2034                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2035                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2036                 FlowEntity natPfibVpnFlowEntity =
2037                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2038                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2039                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2040                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2041             }
2042         }
2043
2044         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2045         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2046         if (ipPortMapping == null) {
2047             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2048             return;
2049         }
2050
2051         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2052             String protocol = intextIpProtocolType.getProtocol().name();
2053             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2054                 String ipPortInternal = ipPortMap.getIpPortInternal();
2055                 String[] ipPortParts = ipPortInternal.split(":");
2056                 if (ipPortParts.length != 2) {
2057                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2058                     return;
2059                 }
2060                 String internalIp = ipPortParts[0];
2061                 String internalPort = ipPortParts[1];
2062
2063                 //Build the flow for the outbound NAPT table
2064                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2065                     + NatConstants.COLON_SEPARATOR + internalPort);
2066                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2067                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2068                 FlowEntity outboundNaptFlowEntity =
2069                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2070
2071                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2072                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2073                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2074
2075                  //Build the flow for the inbound NAPT table
2076                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2077                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2078                 FlowEntity inboundNaptFlowEntity =
2079                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2080
2081                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2082                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2083                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2084             }
2085         }
2086     }
2087
2088     protected void removeNaptFibExternalOutputFlows(Uint32 routerId, Uint64 dpnId, Uuid networkId,
2089                                                     @NonNull Collection<String> externalIps,
2090                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2091             throws ExecutionException, InterruptedException {
2092         Uint32 extVpnId = NatConstants.INVALID_ID;
2093         if (networkId != null) {
2094             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2095             if (vpnUuid != null) {
2096                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2097             } else {
2098                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2099             }
2100         } else {
2101             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2102             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2103         }
2104         if (extVpnId == NatConstants.INVALID_ID) {
2105             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2106             extVpnId = routerId;
2107         }
2108         for (String ip : externalIps) {
2109             String extIp = removeMaskFromIp(ip);
2110             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2111             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2112                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2113                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2114             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2115             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2116         }
2117     }
2118
2119     private String removeMaskFromIp(String ip) {
2120         if (ip != null && !ip.trim().isEmpty()) {
2121             return ip.split("/")[0];
2122         }
2123         return ip;
2124     }
2125
2126     public void removeNaptFlowsFromActiveSwitchInternetVpn(Uint32 routerId, String routerName,
2127                                                            Uint64 dpnId, Uuid networkId, String vpnName,
2128                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2129             throws ExecutionException, InterruptedException {
2130         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2131         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2132
2133         //Remove the NAPT PFIB TABLE entry
2134         Uint32 vpnId = NatConstants.INVALID_ID;
2135         if (vpnName != null) {
2136             // ie called from disassociate vpn case
2137             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2138                     + "with vpnName {}", vpnName);
2139             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2140             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2141                     vpnId);
2142         }
2143
2144         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2145                 networkId, routerName, dpnId)) {
2146             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2147             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2148             FlowEntity natPfibVpnFlowEntity =
2149                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2150             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2151                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2152             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2153
2154             // Remove IP-PORT active NAPT entries and release port from IdManager
2155             // For the router ID get the internal IP , internal port and the corresponding
2156             // external IP and external Port.
2157             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2158             if (ipPortMapping == null) {
2159                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2160                 return;
2161             }
2162             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2163                 String protocol = intextIpProtocolType.getProtocol().name();
2164                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2165                     String ipPortInternal = ipPortMap.getIpPortInternal();
2166                     String[] ipPortParts = ipPortInternal.split(":");
2167                     if (ipPortParts.length != 2) {
2168                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2169                                 + "and port");
2170                         return;
2171                     }
2172                     String internalIp = ipPortParts[0];
2173                     String internalPort = ipPortParts[1];
2174
2175                     //Build the flow for the outbound NAPT table
2176                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2177                             + NatConstants.COLON_SEPARATOR + internalPort);
2178                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2179                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2180                     FlowEntity outboundNaptFlowEntity =
2181                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2182
2183                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2184                             + "active switch with the DPN ID {} and router ID {}",
2185                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2186                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2187
2188                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2189                     final String externalIp = ipPortExternal.getIpAddress();
2190
2191                     //Build the flow for the inbound NAPT table
2192                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2193                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2194                     FlowEntity inboundNaptFlowEntity =
2195                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2196
2197                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2198                             + "active active switch with the DPN ID {} and router ID {}",
2199                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2200                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2201
2202                     // Finally release port from idmanager
2203                     String internalIpPort = internalIp + ":" + internalPort;
2204                     naptManager.removePortFromPool(internalIpPort, externalIp);
2205
2206                     //Remove sessions from models
2207                     naptManager.removeIpPortMappingForRouterID(routerId);
2208                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2209                 }
2210             }
2211         } else {
2212             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2213         }
2214     }
2215
2216     public void removeFlowsFromNonActiveSwitches(Uint32 routerId, String routerName,
2217             Uint64 naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2218             throws ExecutionException, InterruptedException {
2219         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2220
2221         // Remove the flows from the other switches which points to the primary and secondary switches
2222         // for the flows related the router ID.
2223         List<Uint64> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2224         if (allSwitchList.isEmpty()) {
2225             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2226             return;
2227         }
2228         for (Uint64 dpnId : allSwitchList) {
2229             if (!naptSwitchDpnId.equals(dpnId)) {
2230                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2231
2232                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2233                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2234                 FlowEntity preSnatFlowEntity =
2235                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2236
2237                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2238                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2239                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2240
2241                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2242                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2243                     NatUtil.getGroupIdKey(routerName));
2244                 if (groupId != NatConstants.INVALID_ID) {
2245                     LOG.info(
2246                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2247                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2248                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId.longValue());
2249                 } else {
2250                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2251                 }
2252             }
2253         }
2254     }
2255
2256     public void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, @Nullable Uuid networkUuid,
2257                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2258                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2259             throws ExecutionException, InterruptedException {
2260         //Withdraw the corresponding routes from the BGP.
2261         //Get the network ID using the router ID.
2262         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2263                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2264         if (networkUuid == null) {
2265             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2266             return;
2267         }
2268
2269         if (externalIps.isEmpty()) {
2270             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2271             return;
2272         }
2273
2274         if (vpnName == null) {
2275             //Get the VPN Name using the network ID
2276             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2277             if (vpnName == null) {
2278                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2279                     networkUuid, routerId);
2280                 return;
2281             }
2282         }
2283         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2284
2285         //Remove custom FIB routes
2286         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2287         for (String extIp : externalIps) {
2288             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2289         }
2290     }
2291
2292     protected void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2293                                             final Uuid networkUuid, String extGwMacAddress,
2294                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2295             throws ExecutionException, InterruptedException {
2296         clearBgpRoutes(extIp, vpnName);
2297         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2298                 removeFlowInvTx);
2299     }
2300
2301     protected void delFibTsAndReverseTraffic(final Uint64 dpnId, String routerName, Uint32 routerId, String extIp,
2302                                              String vpnName, Uuid extNetworkId, Uint32 tempLabel,
2303                                              String gwMacAddress, boolean switchOver,
2304                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2305             throws ExecutionException, InterruptedException {
2306         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2307         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2308         if (routerName == null) {
2309             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2310             return;
2311         }
2312         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2313         if (extNwProvType == null) {
2314             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2315             return;
2316         }
2317         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2318          * external network provided type is VxLAN
2319          */
2320         if (extNwProvType == ProviderTypes.VXLAN) {
2321             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2322             );
2323             return;
2324         }
2325         if (tempLabel.longValue() < 0) {
2326             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2327                     extIp, routerId);
2328             return;
2329         }
2330         final Uint32 label = tempLabel;
2331         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2332         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2333             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2334             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2335             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2336                     extRouter);
2337
2338             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2339                     externalSubnetId);
2340
2341             if (externalSubnet.isPresent()) {
2342                 vpnName =  externalSubnetId.getValue();
2343             }
2344         }
2345         final String externalVpn = vpnName;
2346         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2347                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2348                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2349         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2350
2351         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2352         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2353         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2354             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2355             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2356         }
2357         if (!switchOver) {
2358             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2359                 Futures.transformAsync(future, result -> {
2360                     //Release label
2361                     if (result.isSuccessful()) {
2362                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2363                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2364                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2365                         return vpnService.removeVpnLabel(labelInput);
2366                     } else {
2367                         String errMsg =
2368                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2369                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2370                         LOG.error(errMsg);
2371                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2372                     }
2373                 }, MoreExecutors.directExecutor());
2374
2375             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2376
2377                 @Override
2378                 public void onFailure(@NonNull Throwable error) {
2379                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2380                         + "got external ip {}", label, extIp, error);
2381                 }
2382
2383                 @Override
2384                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2385                     if (result.isSuccessful()) {
2386                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2387                             + "from VPN {}", externalIp, externalVpn);
2388                     } else {
2389                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2390                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2391                     }
2392                 }
2393             }, MoreExecutors.directExecutor());
2394         } else {
2395             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2396                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2397         }
2398     }
2399
2400     private void delFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2401                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2402                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2403             throws ExecutionException, InterruptedException {
2404         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2405         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2406         if (routerName == null) {
2407             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2408             return;
2409         }
2410         //Get the external network provider type from networkId
2411         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2412         if (extNwProvType == null) {
2413             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2414                     networkUuid);
2415             return;
2416         }
2417         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2418          *  external network provided type is VxLAN
2419          */
2420         if (extNwProvType == ProviderTypes.VXLAN) {
2421             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2422             return;
2423         }
2424         //Get IPMaps from the DB for the router ID
2425         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2426         if (dbIpMaps.isEmpty()) {
2427             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2428             return;
2429         }
2430
2431         Uint32 tempLabel = NatConstants.INVALID_ID;
2432         for (IpMap dbIpMap : dbIpMaps) {
2433             String dbExternalIp = dbIpMap.getExternalIp();
2434             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2435             //Select the IPMap, whose external IP is the IP for which FIB is installed
2436             if (extIp.equals(dbExternalIp)) {
2437                 tempLabel = dbIpMap.getLabel();
2438                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2439                     tempLabel, dbExternalIp, routerId);
2440                 break;
2441             }
2442         }
2443         if (tempLabel == NatConstants.INVALID_ID) {
2444             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2445                     extIp, routerId);
2446             return;
2447         }
2448
2449         final Uint32 label = tempLabel;
2450         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2451         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2452                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2453                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2454         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2455
2456         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2457         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2458         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2459             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2460             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2461         }
2462         if (!switchOver) {
2463             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2464                     Futures.transformAsync(future, result -> {
2465                         //Release label
2466                         if (result.isSuccessful()) {
2467                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2468                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2469                             return vpnService.removeVpnLabel(labelInput);
2470                         } else {
2471                             String errMsg =
2472                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2473                                             + "prefix %s Failed - %s",
2474                                             dpnId, externalIp, result.getErrors());
2475                             LOG.error(errMsg);
2476                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2477                         }
2478                     }, MoreExecutors.directExecutor());
2479
2480             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2481
2482                 @Override
2483                 public void onFailure(@NonNull Throwable error) {
2484                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2485                 }
2486
2487                 @Override
2488                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2489                     if (result.isSuccessful()) {
2490                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2491                             + "from VPN {}", externalIp, vpnName);
2492                     } else {
2493                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2494                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2495                     }
2496                 }
2497             }, MoreExecutors.directExecutor());
2498         } else {
2499             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2500                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2501         }
2502     }
2503
2504     protected void clearFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, Uuid networkUuid,
2505             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2506             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2507         //Withdraw the corresponding routes from the BGP.
2508         //Get the network ID using the router ID.
2509         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2510                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2511         if (networkUuid == null) {
2512             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2513             return;
2514         }
2515
2516         if (externalIps == null || externalIps.isEmpty()) {
2517             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2518             return;
2519         }
2520
2521         if (vpnName == null) {
2522             //Get the VPN Name using the network ID
2523             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2524             if (vpnName == null) {
2525                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2526                     networkUuid, routerId);
2527                 return;
2528             }
2529         }
2530         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2531
2532         //Remove custom FIB routes
2533         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2534         for (String extIp : externalIps) {
2535             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2536                     writeFlowInvTx);
2537         }
2538     }
2539
2540     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2541         //Inform BGP about the route removal
2542         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2543         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2544         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName);
2545     }
2546
2547     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
2548             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2549         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2550         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2551             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2552         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2553     }
2554
2555     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
2556             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2557         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2558         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2559         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2560         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2561     }
2562
2563     /**
2564      * router association to vpn.
2565      *
2566      * @param routerName - Name of router
2567      * @param routerId - router id
2568      * @param bgpVpnName BGP VPN name
2569      */
2570     public void changeLocalVpnIdToBgpVpnId(String routerName, Uint32 routerId, String bgpVpnName,
2571             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2572         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2573         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2574             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2575
2576             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2577
2578             if (bgpVpnId != NatConstants.INVALID_ID) {
2579                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2580                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2581                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2582                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2583                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2584                     getRoutersIdentifier(bgpVpnId), rtrs);
2585
2586                 // Get the allocated Primary NAPT Switch for this router
2587                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2588                         routerId, bgpVpnId);
2589                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2590
2591                 // Get the group ID
2592                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2593                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2594                         extNwProvType);
2595             }
2596         }
2597     }
2598
2599     /**
2600      * router disassociation from vpn.
2601      *
2602      * @param routerName - Name of router
2603      * @param routerId - router id
2604      * @param bgpVpnName BGP VPN name
2605      */
2606     public void changeBgpVpnIdToLocalVpnId(String routerName, Uint32 routerId, String bgpVpnName,
2607             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2608         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2609         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2610             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2611             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2612
2613             // Get the allocated Primary NAPT Switch for this router
2614             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2615
2616             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2617             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2618
2619             // Get the group ID
2620             Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2621             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2622                     writeFlowInvTx, extNwProvType);
2623         }
2624     }
2625
2626     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2627         InstanceIdentifier<Routers> routerInstanceIndentifier =
2628             InstanceIdentifier.builder(ExtRouters.class)
2629                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2630         try {
2631             Optional<Routers> routerData = SingleTransactionDataBroker
2632                     .syncReadOptional(dataBroker,
2633                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2634             return routerData.isPresent() && routerData.get().isEnableSnat();
2635         } catch (ReadFailedException e) {
2636             LOG.error("Failed to read data for router id {}", routerUuid, e);
2637             return false;
2638         }
2639     }
2640
2641     public void installFlowsWithUpdatedVpnId(Uint64 primarySwitchId, String routerName, Uint32 bgpVpnId,
2642                                              Uint32 routerId, boolean isSnatCfgd,
2643                                              TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2644
2645         Uint32 changedVpnId = bgpVpnId;
2646         String idType = "BGP VPN";
2647         if (bgpVpnId == NatConstants.INVALID_ID) {
2648             changedVpnId = routerId;
2649             idType = "router";
2650         }
2651
2652         List<Uint64> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2653         if (switches.isEmpty()) {
2654             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2655             return;
2656         }
2657         for (Uint64 dpnId : switches) {
2658             // Update the BGP VPN ID in the SNAT miss entry to group
2659             if (!dpnId.equals(primarySwitchId)) {
2660                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2661                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2662                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2663                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2664                     NatUtil.getGroupIdKey(routerName));
2665                 if (groupId != NatConstants.INVALID_ID) {
2666                     if (!isSnatCfgd) {
2667                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2668                     }
2669                     LOG.debug(
2670                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2671                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2672                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2673                         groupId, changedVpnId);
2674                     mdsalManager.addFlow(confTx, flowEntity);
2675                 } else {
2676                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2677                 }
2678             } else {
2679                 LOG.debug(
2680                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2681                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2682                 FlowEntity flowEntity =
2683                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2684                 mdsalManager.addFlow(confTx, flowEntity);
2685
2686                 LOG.debug(
2687                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2688                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2689                         idType, changedVpnId, primarySwitchId);
2690                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2691                         changedVpnId, confTx, extNwProvType);
2692
2693                 LOG.debug(
2694                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2695                                 + "which punts the packet to the controller in the Primary switch {}",
2696                         idType, changedVpnId, primarySwitchId);
2697                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2698
2699                 LOG.debug(
2700                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2701                                 + " outgoing packet to FIB Table in the Primary switch {}",
2702                         idType, changedVpnId, primarySwitchId);
2703                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2704
2705                 LOG.debug(
2706                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2707                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2708                                 + " {}", idType, changedVpnId, primarySwitchId);
2709                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2710
2711                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2712                         primarySwitchId);
2713                 Uint32 vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2714                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2715                 if (vpnId != NatConstants.INVALID_ID) {
2716                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2717                 }
2718             }
2719         }
2720     }
2721
2722     public void updateNaptFlowsWithVpnId(Uint64 dpnId, String routerName, Uint32 routerId, Uint32 bgpVpnId) {
2723         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2724         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2725         if (ipPortMapping == null) {
2726             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2727             return;
2728         }
2729         // Get the External Gateway MAC Address
2730         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2731         if (extGwMacAddress != null) {
2732             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2733                     extGwMacAddress, routerId);
2734         } else {
2735             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2736                     routerId);
2737             return;
2738         }
2739         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2740             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2741                 String ipPortInternal = ipPortMap.getIpPortInternal();
2742                 String[] ipPortParts = ipPortInternal.split(":");
2743                 if (ipPortParts.length != 2) {
2744                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2745                     return;
2746                 }
2747                 String internalIp = ipPortParts[0];
2748                 String internalPort = ipPortParts[1];
2749                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2750                         internalIp, internalPort);
2751                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2752                 NAPTEntryEvent.Protocol protocol;
2753                 switch (protocolTypes) {
2754                     case TCP:
2755                         protocol = NAPTEntryEvent.Protocol.TCP;
2756                         break;
2757                     case UDP:
2758                         protocol = NAPTEntryEvent.Protocol.UDP;
2759                         break;
2760                     default:
2761                         protocol = NAPTEntryEvent.Protocol.TCP;
2762                 }
2763                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2764                 SessionAddress externalAddress =
2765                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2766                 Uint32 internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2767                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2768                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2769                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2770                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2771             }
2772         }
2773     }
2774
2775     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName, Uint32 groupId,
2776                                                           Uint32 changedVpnId) {
2777
2778         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2779             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2780         List<MatchInfo> matches = new ArrayList<>();
2781         matches.add(MatchEthernetType.IPV4);
2782         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2783                 MetaDataUtil.METADATA_MASK_VRFID));
2784
2785         List<ActionInfo> actionsInfo = new ArrayList<>();
2786         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2787                 elanManager, idManager, changedVpnId, routerName);
2788         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
2789         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2790                 actionsInfo);
2791         actionsInfo.add(new ActionGroup(groupId.longValue()));
2792         List<InstructionInfo> instructions = new ArrayList<>();
2793         instructions.add(new InstructionApplyActions(actionsInfo));
2794         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2795         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2796             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2797             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2798
2799         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2800         return flowEntity;
2801     }
2802
2803     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(Uint64 dpId, String routerName,
2804                                                                         Uint32 changedVpnId) {
2805
2806         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2807                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2808         List<MatchInfo> matches = new ArrayList<>();
2809         matches.add(MatchEthernetType.IPV4);
2810         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2811                 MetaDataUtil.METADATA_MASK_VRFID));
2812
2813         List<InstructionInfo> instructions = new ArrayList<>();
2814         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2815
2816         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2817         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2818             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2819             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2820
2821         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2822         return flowEntity;
2823     }
2824
2825     // TODO : Replace this with ITM Rpc once its available with full functionality
2826     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(Uint64 dpnId, String routerName,
2827                                                                      Uint32 routerId, Uint32 changedVpnId,
2828                                                                      TypedWriteTransaction<Configuration> confTx,
2829                                                                      ProviderTypes extNwProvType) {
2830
2831         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2832             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2833         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2834                 extNwProvType);
2835         mdsalManager.addFlow(confTx, flowEntity);
2836     }
2837
2838     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName,
2839                                                          Uint32 routerIdLongVal, Uint32 changedVpnId,
2840                                                          ProviderTypes extNwProvType) {
2841         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2842             dpId, routerName, changedVpnId);
2843         List<MatchInfo> matches = new ArrayList<>();
2844         matches.add(MatchEthernetType.IPV4);
2845
2846         Uint64 tunnelId = Uint64.valueOf(changedVpnId);
2847         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2848             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2849         }
2850         matches.add(new MatchTunnelId(tunnelId));
2851
2852         List<InstructionInfo> instructions = new ArrayList<>();
2853         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2854             MetaDataUtil.METADATA_MASK_VRFID));
2855         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2856         Uint32 routerId = routerIdLongVal;
2857         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
2858         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2859             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2860             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2861         return flowEntity;
2862     }
2863
2864     public void createOutboundTblEntryWithBgpVpn(Uint64 dpnId, Uint32 routerId, Uint32 changedVpnId,
2865                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2866         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2867             dpnId, routerId, changedVpnId);
2868         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2869                 NwConstants.IP_PROT_TCP);
2870         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2871         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2872
2873         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2874                 NwConstants.IP_PROT_UDP);
2875         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2876         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2877
2878         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2879         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2880         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2881     }
2882
2883     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(Uint64 dpId, Uint32 routerId,
2884                                                            Uint32 changedVpnId, int protocol) {
2885         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2886             dpId, routerId, changedVpnId);
2887         Uint64 cookie = getCookieOutboundFlow(routerId);
2888         List<MatchInfo> matches = new ArrayList<>();
2889         matches.add(MatchEthernetType.IPV4);
2890         matches.add(new MatchIpProtocol((short)protocol));
2891         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2892                 MetaDataUtil.METADATA_MASK_VRFID));
2893
2894         List<InstructionInfo> instructions = new ArrayList<>();
2895         List<ActionInfo> actionsInfos = new ArrayList<>();
2896         actionsInfos.add(new ActionPuntToController());
2897         if (snatPuntTimeout != 0) {
2898             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2899         }
2900         instructions.add(new InstructionApplyActions(actionsInfos));
2901
2902         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2903         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2904             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2905         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2906         return flowEntity;
2907     }
2908
2909     public void installNaptPfibEntryWithBgpVpn(Uint64 dpnId, Uint32 segmentId, Uint32 changedVpnId,
2910                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2911         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2912             dpnId, segmentId, changedVpnId);
2913         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2914         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2915     }
2916
2917     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(Uint64 dpId, Uint32 segmentId, Uint32 changedVpnId) {
2918
2919         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2920             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2921         List<MatchInfo> matches = new ArrayList<>();
2922         matches.add(MatchEthernetType.IPV4);
2923         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2924                 MetaDataUtil.METADATA_MASK_VRFID));
2925
2926         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2927         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2928         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
2929         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2930         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2931
2932         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2933         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2934             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2935             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2936         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2937         return flowEntity;
2938     }
2939
2940     @Override
2941     protected ExternalRoutersListener getDataTreeChangeListener() {
2942         return ExternalRoutersListener.this;
2943     }
2944
2945     protected void installNaptPfibEntriesForExternalSubnets(String routerName, Uint64 dpnId,
2946                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2947         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2948                 routerName);
2949         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2950             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2951             if (subnetVpnId != NatConstants.INVALID_ID) {
2952                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2953                     + "and vpnId {}", dpnId, subnetVpnId);
2954                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2955             }
2956         }
2957     }
2958 }