natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 145 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 146 import org.opendaylight.yangtools.yang.common.RpcResult;
 147 import org.opendaylight.yangtools.yang.common.Uint32;
 148 import org.opendaylight.yangtools.yang.common.Uint64;
 149 import org.slf4j.Logger;
 150 import org.slf4j.LoggerFactory;
 151
 152
 153 @Singleton
 154 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 155     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 156
 157     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 158     private static final Uint64 COOKIE_VM_LFIB_TABLE = Uint64.valueOf("8000022", 16).intern();
 159
 160     private final DataBroker dataBroker;
 161     private final ManagedNewTransactionRunner txRunner;
 162     private final IMdsalApiManager mdsalManager;
 163     private final ItmRpcService itmManager;
 164     private final OdlInterfaceRpcService odlInterfaceRpcService;
 165     private final IdManagerService idManager;
 166     private final NaptManager naptManager;
 167     private final NAPTSwitchSelector naptSwitchSelector;
 168     private final IBgpManager bgpManager;
 169     private final VpnRpcService vpnService;
 170     private final FibRpcService fibService;
 171     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 172     private final NaptEventHandler naptEventHandler;
 173     private final NaptPacketInHandler naptPacketInHandler;
 174     private final IFibManager fibManager;
 175     private final IVpnManager vpnManager;
 176     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 177     private final NatMode natMode;
 178     private final IElanService elanManager;
 179     private final JobCoordinator coordinator;
 180     private final IInterfaceManager interfaceManager;
 181     private final NatOverVxlanUtil natOverVxlanUtil;
 182     private final int snatPuntTimeout;
 183
 184     @Inject
 185     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 186                                    final ItmRpcService itmManager,
 187                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 188                                    final IdManagerService idManager,
 189                                    final NaptManager naptManager,
 190                                    final NAPTSwitchSelector naptSwitchSelector,
 191                                    final IBgpManager bgpManager,
 192                                    final VpnRpcService vpnService,
 193                                    final FibRpcService fibService,
 194                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 195                                    final NaptEventHandler naptEventHandler,
 196                                    final NaptPacketInHandler naptPacketInHandler,
 197                                    final IFibManager fibManager,
 198                                    final IVpnManager vpnManager,
 199                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 200                                    final NatserviceConfig config,
 201                                    final IElanService elanManager,
 202                                    final JobCoordinator coordinator,
 203                                    final NatOverVxlanUtil natOverVxlanUtil,
 204                                    final IInterfaceManager interfaceManager) {
 205         super(Routers.class, ExternalRoutersListener.class);
 206         this.dataBroker = dataBroker;
 207         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 208         this.mdsalManager = mdsalManager;
 209         this.itmManager = itmManager;
 210         this.odlInterfaceRpcService = odlInterfaceRpcService;
 211         this.idManager = idManager;
 212         this.naptManager = naptManager;
 213         this.naptSwitchSelector = naptSwitchSelector;
 214         this.bgpManager = bgpManager;
 215         this.vpnService = vpnService;
 216         this.fibService = fibService;
 217         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 218         this.naptEventHandler = naptEventHandler;
 219         this.naptPacketInHandler = naptPacketInHandler;
 220         this.fibManager = fibManager;
 221         this.vpnManager = vpnManager;
 222         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 223         this.elanManager = elanManager;
 224         this.coordinator = coordinator;
 225         this.interfaceManager = interfaceManager;
 226         this.natOverVxlanUtil = natOverVxlanUtil;
 227         if (config != null) {
 228             this.natMode = config.getNatMode();
 229             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 230         } else {
 231             this.natMode = NatMode.Controller;
 232             this.snatPuntTimeout = 0;
 233         }
 234     }
 235
 236     @Override
 237     @PostConstruct
 238     public void init() {
 239         LOG.info("{} init", getClass().getSimpleName());
 240         // This class handles ExternalRouters for Controller SNAT mode.
 241         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 242         if (natMode == NatMode.Controller) {
 243             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 244             NatUtil.createGroupIdPool(idManager);
 245         }
 246     }
 247
 248     @Override
 249     protected InstanceIdentifier<Routers> getWildCardPath() {
 250         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 251     }
 252
 253     @Override
 254     // TODO Clean up the exception handling
 255     @SuppressWarnings("checkstyle:IllegalCatch")
 256     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 257         // Populate the router-id-name container
 258         String routerName = routers.getRouterName();
 259         LOG.info("add : external router event for {}", routerName);
 260         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
 261         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 262         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 263         try {
 264             if (routers.isEnableSnat()) {
 265                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 266                     () -> Collections.singletonList(
 267                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 268                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 269                             Uint32 bgpVpnId = NatConstants.INVALID_ID;
 270                             if (bgpVpnUuid != null) {
 271                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 272                             }
 273                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 274                             // Allocate Primary Napt Switch for this router
 275                             Uint64 primarySwitchId = getPrimaryNaptSwitch(routerName);
 276                             if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 277                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 278                             }
 279                         }
 280                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 281             } else {
 282                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 283             }
 284         } catch (Exception ex) {
 285             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 286                     routerName, ex);
 287         }
 288     }
 289
 290     public void handleEnableSnat(Routers routers, Uint32 routerId, Uint64 primarySwitchId, Uint32 bgpVpnId,
 291                                  TypedWriteTransaction<Configuration> confTx) {
 292         String routerName = routers.getRouterName();
 293         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 294
 295         naptManager.initialiseExternalCounter(routers, routerId);
 296         subnetRegisterMapping(routers, routerId);
 297
 298         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 299             primarySwitchId, routerName);
 300
 301         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 302                 routers.getNetworkId());
 303         if (extNwProvType == null) {
 304             LOG.error("handleEnableSnat : External Network Provider Type missing");
 305             return;
 306         }
 307
 308         if (bgpVpnId != NatConstants.INVALID_ID) {
 309             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId,
 310                 routers.getNetworkId(),false, confTx, extNwProvType);
 311         } else {
 312             // write metadata and punt
 313             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 314             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, routers.getNetworkId(), confTx);
 315             // Now install entries in SNAT tables to point to Primary for each router
 316             List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 317             for (Uint64 dpnId : switches) {
 318                 // Handle switches and NAPT switches separately
 319                 if (!dpnId.equals(primarySwitchId)) {
 320                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 321                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 322                 }
 323             }
 324         }
 325
 326         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 327         if (externalIps.isEmpty()) {
 328             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 329             return;
 330         } else {
 331             for (String externalIpAddrPrefix : externalIps) {
 332                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 333                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 334                 externalIpAddrPrefix = NatUtil.validateAndAddNetworkMask(externalIpAddrPrefix);
 335                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 336                 );
 337             }
 338         }
 339         LOG.debug("handleEnableSnat : Exit");
 340     }
 341
 342     private Uint64 getPrimaryNaptSwitch(String routerName) {
 343         // Allocate Primary Napt Switch for this router
 344         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 345         if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 346             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 347                 primarySwitchId, routerName);
 348             return primarySwitchId;
 349         }
 350         return selectNewNAPTSwitch(routerName);
 351     }
 352
 353     private Uint64 selectNewNAPTSwitch(String routerName) {
 354         // Allocated an id from VNI pool for the Router.
 355         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 356         Uint64 primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName, null);
 357         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 358
 359         return primarySwitchId;
 360     }
 361
 362     protected void installNaptPfibExternalOutputFlow(String routerName, Uint32 routerId, Uint64 dpnId,
 363                                                      TypedWriteTransaction<Configuration> confTx) {
 364         Uint32 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 365         if (extVpnId == NatConstants.INVALID_ID) {
 366             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 367             extVpnId = routerId;
 368         }
 369         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 370         if (externalIps.isEmpty()) {
 371             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 372                 dpnId, extVpnId);
 373             return;
 374         }
 375         for (String ip : externalIps) {
 376             Uuid subnetId = getSubnetIdForFixedIp(ip);
 377             if (subnetId != null) {
 378                 Uint32 subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 379                 if (subnetVpnId != NatConstants.INVALID_ID) {
 380                     extVpnId = subnetVpnId;
 381                 }
 382                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 383                     dpnId, extVpnId, subnetId);
 384                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 385                 if (postNaptFlowEntity != null) {
 386                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 387                 }
 388             }
 389         }
 390     }
 391
 392     @Nullable
 393     private Uuid getSubnetIdForFixedIp(String ip) {
 394         if (ip != null) {
 395             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 396             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 397             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 398         }
 399         LOG.error("getSubnetIdForFixedIp : ip is null");
 400         return null;
 401     }
 402
 403     protected void subnetRegisterMapping(Routers routerEntry, Uint32 segmentId) {
 404         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 405         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 406         int counter = 0;
 407         int extIpCounter = externalIps.size();
 408         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 409                 counter, extIpCounter);
 410         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 411         if (subnetIds == null) {
 412             return;
 413         }
 414         for (Uuid subnet : subnetIds) {
 415             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 416             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 417                 .builder(Subnetmaps.class)
 418                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 419                 .build();
 420             Optional<Subnetmap> sn;
 421             try {
 422                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 423                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 424             } catch (ReadFailedException e) {
 425                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 426                 sn = Optional.absent();
 427             }
 428             if (sn.isPresent()) {
 429                 // subnets
 430                 Subnetmap subnetmapEntry = sn.get();
 431                 String subnetString = subnetmapEntry.getSubnetIp();
 432                 String[] subnetSplit = subnetString.split("/");
 433                 String subnetIp = subnetSplit[0];
 434                 try {
 435                     InetAddress address = InetAddress.getByName(subnetIp);
 436                     if (address instanceof Inet6Address) {
 437                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 438                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 439                         continue;
 440                     }
 441                 } catch (UnknownHostException e) {
 442                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 443                     return;
 444                 }
 445                 String subnetPrefix = "0";
 446                 if (subnetSplit.length == 2) {
 447                     subnetPrefix = subnetSplit[1];
 448                 }
 449                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 450                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 451                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 452                 //externalIps
 453                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 454                         counter, extIpCounter);
 455                 if (extIpCounter != 0) {
 456                     if (counter < extIpCounter) {
 457                         String[] ipSplit = externalIps.get(counter).split("/");
 458                         String externalIp = ipSplit[0];
 459                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 460                         if (ipSplit.length == 2) {
 461                             extPrefix = ipSplit[1];
 462                         }
 463                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 464                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 465                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 466                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 467                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 468                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 469                     } else {
 470                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 471                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 472                         String[] ipSplit = externalIps.get(counter).split("/");
 473                         String externalIp = ipSplit[0];
 474                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 475                         if (ipSplit.length == 2) {
 476                             extPrefix = ipSplit[1];
 477                         }
 478                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 479                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 480                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 481                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 482                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 483                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 484                             externalIp, extPrefix);
 485                     }
 486                 }
 487                 counter++;
 488                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 489             } else {
 490                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 491             }
 492         }
 493     }
 494
 495     private void addOrDelDefFibRouteToSNAT(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 496             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 497             throws ExecutionException, InterruptedException {
 498         //Check if BGP VPN exists. If exists then invoke the new method.
 499         if (bgpVpnId != NatConstants.INVALID_ID) {
 500             if (bgpVpnUuid != null) {
 501                 String bgpVpnName = bgpVpnUuid.getValue();
 502                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 503                     bgpVpnId, bgpVpnName);
 504                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 505                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 506                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 507             }
 508             if (create) {
 509                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 510             } else {
 511                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 512             }
 513             return;
 514         }
 515
 516         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 517         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 518     }
 519
 520     private void addOrDelDefaultFibRouteForSNAT(String routerName, Uint32 routerId, boolean create,
 521             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 522         List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 523         if (switches.isEmpty()) {
 524             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 525             return;
 526         }
 527         if (routerId == NatConstants.INVALID_ID) {
 528             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 529                     + "default NAT route in FIB", routerName);
 530             return;
 531         }
 532         for (Uint64 dpnId : switches) {
 533             if (create) {
 534                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 535                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 536                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 537             } else {
 538                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 539                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 540                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 541             }
 542         }
 543     }
 544
 545     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId,
 546                                                      Uint32 bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 547         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 548         if (dpnIds.isEmpty()) {
 549             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 550                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 551             return;
 552         }
 553         for (Uint64 dpnId : dpnIds) {
 554             if (bgpVpnId != NatConstants.INVALID_ID) {
 555                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 556                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 557                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 558             } else {
 559                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 560                     + "in dpn {} for the internal vpn", routerId, dpnId);
 561                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 562             }
 563         }
 564     }
 565
 566     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 567                                                         TypedReadWriteTransaction<Configuration> confTx)
 568             throws ExecutionException, InterruptedException {
 569         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 570         if (dpnIds.isEmpty()) {
 571             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 572                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 573             return;
 574         }
 575         for (Uint64 dpnId : dpnIds) {
 576             if (bgpVpnId != NatConstants.INVALID_ID) {
 577                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 578                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 579                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 580             } else {
 581                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 582                     + "in dpn {} for the internal vpn", routerId, dpnId);
 583                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 584             }
 585         }
 586     }
 587
 588     protected void installOutboundMissEntry(String routerName, Uint32 routerId, Uint64 primarySwitchId,
 589         TypedWriteTransaction<Configuration> confTx) {
 590         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 591         if (routerId != NatConstants.INVALID_ID) {
 592             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 593                     primarySwitchId, routerId);
 594             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 595         } else {
 596             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 597                 + "createAndInstallMissEntry", routerName);
 598         }
 599     }
 600
 601     public String getFlowRefOutbound(Uint64 dpnId, short tableId, Uint32 routerID, int protocol) {
 602         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 603                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 604     }
 605
 606     private String getFlowRefNaptPreFib(Uint64 dpnId, short tableId, Uint32 vpnId) {
 607         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 608                 .FLOWID_SEPARATOR + vpnId;
 609     }
 610
 611     public Uint64 getCookieOutboundFlow(Uint32 routerId) {
 612         return Uint64.valueOf((NwConstants.COOKIE_OUTBOUND_NAPT_TABLE).toJava().add(
 613                 new BigInteger("0110001", 16)).add(BigInteger.valueOf(routerId.longValue())));
 614     }
 615
 616     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, Uint64 cookie) {
 617         long l4SrcPortField;
 618         long l4DstPortField;
 619         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 620
 621         if (protocol == NwConstants.IP_PROT_TCP) {
 622             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 623             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 624         } else {
 625             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 626             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 627         }
 628         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 629                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 630                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 631                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 632                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 633                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 634                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 635                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 636                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 637                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 638                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 639                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 640                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 641                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 642                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 643
 644         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 645                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 646     }
 647
 648     private FlowEntity buildIcmpDropFlow(Uint64 dpnId, Uint32 routerId, Uint32 vpnId) {
 649         List<MatchInfo> matches = new ArrayList<>();
 650         matches.add(MatchEthernetType.IPV4);
 651         matches.add(MatchIpProtocol.ICMP);
 652         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 653                 MetaDataUtil.METADATA_MASK_VRFID));
 654
 655         List<ActionInfo> actionInfos = new ArrayList<>();
 656         actionInfos.add(new ActionDrop());
 657
 658         List<InstructionInfo> instructions = new ArrayList<>();
 659         instructions.add(new InstructionApplyActions(actionInfos));
 660
 661         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 662                 NwConstants.IP_PROT_ICMP);
 663         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 664                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 665                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 666         return flow;
 667     }
 668
 669     protected FlowEntity buildOutboundFlowEntity(Uint64 dpId, Uint32 routerId, int protocol) {
 670         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 671         Uint64 cookie = getCookieOutboundFlow(routerId);
 672         List<MatchInfo> matches = new ArrayList<>();
 673         matches.add(MatchEthernetType.IPV4);
 674         matches.add(new MatchIpProtocol((short)protocol));
 675         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 676                 MetaDataUtil.METADATA_MASK_VRFID));
 677
 678         List<InstructionInfo> instructions = new ArrayList<>();
 679         List<ActionInfo> actionsInfos = new ArrayList<>();
 680         actionsInfos.add(new ActionPuntToController());
 681         if (snatPuntTimeout != 0) {
 682             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 683         }
 684         instructions.add(new InstructionApplyActions(actionsInfos));
 685
 686         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 687         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 688             5, flowRef, 0, 0,
 689             cookie, matches, instructions);
 690         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 691         return flowEntity;
 692     }
 693
 694     public void createOutboundTblEntry(Uint64 dpnId, Uint32 routerId, TypedWriteTransaction<Configuration> confTx) {
 695         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 696         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 697         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 698         mdsalManager.addFlow(confTx, tcpflowEntity);
 699
 700         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 701         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 702         mdsalManager.addFlow(confTx, udpflowEntity);
 703
 704         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 705         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 706         mdsalManager.addFlow(confTx, icmpDropFlow);
 707     }
 708
 709     @Nullable
 710     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 711         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 712         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 713         try {
 714             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 715                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 716                     .setSourceDpid(srcDpId)
 717                     .setDestinationDpid(dstDpId)
 718                     .setTunnelType(tunType)
 719                     .build());
 720             rpcResult = result.get();
 721             if (!rpcResult.isSuccessful()) {
 722                 tunType = TunnelTypeGre.class;
 723                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 724                     .setSourceDpid(srcDpId)
 725                     .setDestinationDpid(dstDpId)
 726                     .setTunnelType(tunType)
 727                     .build());
 728                 rpcResult = result.get();
 729                 if (!rpcResult.isSuccessful()) {
 730                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 731                             rpcResult.getErrors());
 732                 } else {
 733                     return rpcResult.getResult().getInterfaceName();
 734                 }
 735                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 736                         rpcResult.getErrors());
 737             } else {
 738                 return rpcResult.getResult().getInterfaceName();
 739             }
 740         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 741             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 742                     + "between {} and {}", srcDpId, dstDpId, e);
 743         }
 744
 745         return null;
 746     }
 747
 748     protected void installSnatMissEntryForPrimrySwch(Uint64 dpnId, String routerName, Uint32 routerId,
 749         TypedWriteTransaction<Configuration> confTx) {
 750
 751         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 752         // Install miss entry pointing to group
 753         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 754         mdsalManager.addFlow(confTx, flowEntity);
 755     }
 756
 757     protected void installSnatMissEntry(Uint64 dpnId, List<BucketInfo> bucketInfo,
 758             String routerName, Uint32 routerId) {
 759         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 760             dpnId, bucketInfo.get(0));
 761         // Install the select group
 762         Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 763             NatUtil.getGroupIdKey(routerName));
 764         if (groupId == NatConstants.INVALID_ID) {
 765             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 766             return;
 767         }
 768         GroupEntity groupEntity =
 769             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 770         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 771         mdsalManager.syncInstallGroup(groupEntity);
 772         // Install miss entry pointing to group
 773         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 774         if (flowEntity == null) {
 775             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 776                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 777                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 778             return;
 779         }
 780         mdsalManager.installFlow(flowEntity);
 781     }
 782
 783     void installGroup(Uint64 dpnId, String routerName, Uint32 groupId, List<BucketInfo> bucketInfo) {
 784         GroupEntity groupEntity =
 785             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 786         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 787         mdsalManager.syncInstallGroup(groupEntity);
 788     }
 789
 790     private FlowEntity buildSnatFlowEntity(Uint64 dpId, String routerName, Uint32 routerId, Uint32 groupId) {
 791         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 792                 dpId, routerName, groupId);
 793         List<MatchInfo> matches = new ArrayList<>();
 794         matches.add(MatchEthernetType.IPV4);
 795         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 796                 MetaDataUtil.METADATA_MASK_VRFID));
 797
 798         List<ActionInfo> actionsInfo = new ArrayList<>();
 799         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 800                 routerId, routerName);
 801         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
 802         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 803         actionsInfo.add(new ActionGroup(groupId.longValue()));
 804         List<InstructionInfo> instructions = new ArrayList<>();
 805         instructions.add(new InstructionApplyActions(actionsInfo));
 806         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 807         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 808                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 809                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 810
 811         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 812         return flowEntity;
 813     }
 814
 815     private FlowEntity buildSnatFlowEntityForPrmrySwtch(Uint64 dpId, String routerName, Uint32 routerId) {
 816
 817         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 818             routerName);
 819         List<MatchInfo> matches = new ArrayList<>();
 820         matches.add(MatchEthernetType.IPV4);
 821         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 822                 MetaDataUtil.METADATA_MASK_VRFID));
 823
 824         List<InstructionInfo> instructions = new ArrayList<>();
 825         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 826
 827         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 828         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 829             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 830             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 831
 832         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 833         return flowEntity;
 834     }
 835
 836     // TODO : Replace this with ITM Rpc once its available with full functionality
 837     protected void installTerminatingServiceTblEntry(Uint64 dpnId, String routerName, Uint32 routerId,
 838         TypedWriteTransaction<Configuration> confTx) {
 839
 840         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 841             dpnId, routerName);
 842         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 843         if (flowEntity == null) {
 844             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 845                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 846                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 847                     dpnId, routerName);
 848             return;
 849         }
 850         mdsalManager.addFlow(confTx, flowEntity);
 851
 852     }
 853
 854     private FlowEntity buildTsFlowEntity(Uint64 dpId, String routerName, Uint32 routerId) {
 855         List<MatchInfo> matches = new ArrayList<>();
 856         matches.add(MatchEthernetType.IPV4);
 857         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 858                 idManager, routerId, routerName);
 859         matches.add(new MatchTunnelId(tunnelId));
 860         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
 861         List<InstructionInfo> instructions = new ArrayList<>();
 862         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 863                 MetaDataUtil.METADATA_MASK_VRFID));
 864         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 865         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 866                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
 867                 NwConstants.COOKIE_TS_TABLE, matches, instructions);
 868         return flowEntity;
 869     }
 870
 871     public String getFlowRefTs(Uint64 dpnId, short tableId, Uint32 routerID) {
 872         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 873                 .FLOWID_SEPARATOR + routerID;
 874     }
 875
 876     public static String getFlowRefSnat(Uint64 dpnId, short tableId, String routerID) {
 877         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 878                 .FLOWID_SEPARATOR + routerID;
 879     }
 880
 881     protected void handleSwitches(Uint64 dpnId, String routerName, Uint32 routerId, Uint64 primarySwitchId) {
 882         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 883         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 884         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 885         List<BucketInfo> listBucketInfo = new ArrayList<>();
 886
 887         if (ifNamePrimary != null) {
 888             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 889             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 890                     interfaceManager, ifNamePrimary, routerId, true);
 891             if (listActionInfoPrimary.isEmpty()) {
 892                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 893                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 894                         ifNamePrimary);
 895             }
 896         } else {
 897             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 898                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 899         }
 900         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 901
 902         listBucketInfo.add(0, bucketPrimary);
 903         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 904     }
 905
 906     List<BucketInfo> getBucketInfoForNonNaptSwitches(Uint64 nonNaptSwitchId,
 907                                                      Uint64 primarySwitchId, String routerName, Uint32 routerId) {
 908         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 909         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 910         List<BucketInfo> listBucketInfo = new ArrayList<>();
 911
 912         if (ifNamePrimary != null) {
 913             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 914                     ifNamePrimary);
 915             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 916                     interfaceManager, ifNamePrimary, routerId, true);
 917             if (listActionInfoPrimary.isEmpty()) {
 918                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 919                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 920                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 921             }
 922         } else {
 923             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 924                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 925         }
 926         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 927
 928         listBucketInfo.add(0, bucketPrimary);
 929         return listBucketInfo;
 930     }
 931
 932     protected void handlePrimaryNaptSwitch(Uint64 dpnId, String routerName, Uint32 routerId, Uuid externalNwUuid,
 933         TypedWriteTransaction<Configuration> confTx) {
 934
 935        /*
 936         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 937         */
 938
 939         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 940
 941 /*
 942         List<BucketInfo> listBucketInfo = new ArrayList<>();
 943         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 944         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 945         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 946         listBucketInfo.add(0, bucketPrimary);
 947 */
 948
 949         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 950         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 951         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 952         installNaptPfibEntry(dpnId, routerId, confTx);
 953         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 954         if (networkId != null) {
 955             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 956             if (vpnUuid != null) {
 957                 Uint32 extVpnId = NatUtil.getExternalVpnIdForExtNetwork(dataBroker, externalNwUuid);
 958                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 959                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 960                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 961                     if (extVpnId != null && extVpnId != NatConstants.INVALID_ID) {
 962                         installNaptPfibEntry(dpnId, extVpnId, null);
 963                     }
 964                     return Collections.emptyList();
 965                 });
 966             } else {
 967                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 968             }
 969         } else {
 970             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 971         }
 972     }
 973
 974     public void installNaptPfibEntry(Uint64 dpnId, Uint32 segmentId,
 975             @Nullable TypedWriteTransaction<Configuration> confTx) {
 976         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 977         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 978         if (confTx != null) {
 979             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 980         } else {
 981             mdsalManager.installFlow(naptPfibFlowEntity);
 982         }
 983     }
 984
 985     public FlowEntity buildNaptPfibFlowEntity(Uint64 dpId, Uint32 segmentId) {
 986
 987         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 988         List<MatchInfo> matches = new ArrayList<>();
 989         matches.add(MatchEthernetType.IPV4);
 990         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId.longValue()),
 991                 MetaDataUtil.METADATA_MASK_VRFID));
 992
 993         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 994         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 995         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
 996         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 997         instructionInfo.add(new InstructionApplyActions(listActionInfo));
 998
 999         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1000         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1001             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1002             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1003         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1004         return flowEntity;
1005     }
1006
1007     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, Uint64 dpnId, Routers router,
1008                                          Uint32 routerId, String routerName, String externalIp) {
1009         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1010             dpnId, routerId, externalIp);
1011         Uuid networkId = router.getNetworkId();
1012         if (networkId == null) {
1013             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1014             return;
1015         }
1016         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1017         if (vpnName == null) {
1018             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1019                     + "configuration {} in router {}", networkId, externalIp, routerId);
1020             return;
1021         }
1022         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1023             externalIp, networkId, router, confTx);
1024         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1025             dpnId, routerId, externalIp);
1026     }
1027
1028     public void advToBgpAndInstallFibAndTsFlows(final Uint64 dpnId, final short tableId, final String vpnName,
1029                                                 final Uint32 routerId, final String routerName, final String externalIp,
1030                                                 final Uuid extNetworkId, @Nullable final Routers router,
1031                                                 final TypedWriteTransaction<Configuration> confTx) {
1032         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1033                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1034         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1035         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1036         if (rd == null || rd.isEmpty()) {
1037             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1038             return;
1039         }
1040         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1041         if (extNwProvType == null) {
1042             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1043             return;
1044         }
1045         if (extNwProvType == ProviderTypes.VXLAN) {
1046             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1047                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1048             return;
1049         }
1050         //Generate VPN label for the external IP
1051         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1052             .setIpPrefix(externalIp).build();
1053         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1054
1055         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1056         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1057             if (result.isSuccessful()) {
1058                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1059                 GenerateVpnLabelOutput output = result.getResult();
1060                 final Uint32 label = output.getLabel();
1061
1062                 int externalIpInDsFlag = 0;
1063                 //Get IPMaps from the DB for the router ID
1064                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1065                 for (IpMap dbIpMap : dbIpMaps) {
1066                     String dbExternalIp = dbIpMap.getExternalIp();
1067                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1068                     if (dbExternalIp.contains(externalIp)) {
1069                         String dbInternalIp = dbIpMap.getInternalIp();
1070                         IpMapKey dbIpMapKey = dbIpMap.key();
1071                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1072                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1073                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1074                             .setExternalIp(dbExternalIp).setLabel(label).build();
1075                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1076                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1077                         externalIpInDsFlag++;
1078                     }
1079                 }
1080                 if (externalIpInDsFlag <= 0) {
1081                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1082                             + "Failed to update label {} for routerId {} in DS",
1083                             externalIp, label, routerId);
1084                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1085                         + " found in DS for router %s", label, externalIp, routerId);
1086                     return Futures.immediateFailedFuture(new Exception(errMsg));
1087                 }
1088                 //Inform BGP
1089                 Uint32 l3vni = Uint32.ZERO;
1090                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1091                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
1092                 }
1093                 Routers extRouter = router != null ? router :
1094                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1095                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1096                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1097                     RouteOrigin.STATIC, dpnId);
1098
1099                 //Install custom FIB routes
1100                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1101                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1102                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1103                         extNwProvType);
1104                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1105
1106                 //Install custom FIB routes - FIB table.
1107                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1108                         routerName, externalIp);
1109                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1110                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1111                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1112                 }
1113                 String externalVpn = vpnName;
1114                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1115                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1116                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1117                             externalSubnetId);
1118                     if (externalSubnet.isPresent()) {
1119                         externalVpn =  externalSubnetId.getValue();
1120                     }
1121                 }
1122                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1123                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1124                     .setVpnName(externalVpn)
1125                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1126                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1127                     .setInstruction(fibTableCustomInstructions).build();
1128                 return fibService.createFibEntry(input);
1129             } else {
1130                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1131                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1132                     externalIp, vpnName, result.getErrors());
1133                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1134             }
1135         }, MoreExecutors.directExecutor());
1136
1137         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1138
1139             @Override
1140             public void onFailure(@NonNull Throwable error) {
1141                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1142             }
1143
1144             @Override
1145             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1146                 if (result.isSuccessful()) {
1147                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1148                             externalIp);
1149                 } else {
1150                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1151                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1152                 }
1153             }
1154         }, MoreExecutors.directExecutor());
1155     }
1156
1157     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1158             String externalIp) {
1159         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1160         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1161         Uint32 externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1162                 externalIp, router);
1163         int instructionIndex = 0;
1164         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1165             Uint64 subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId.longValue());
1166             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1167                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1168             instructionIndex++;
1169         }
1170
1171         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1172         return fibTableCustomInstructions;
1173     }
1174
1175     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, short tableId,
1176         TypedWriteTransaction<Configuration> confTx) {
1177         List<MatchInfo> matches = new ArrayList<>();
1178         matches.add(MatchEthernetType.MPLS_UNICAST);
1179         matches.add(new MatchMplsLabel(serviceId.longValue()));
1180
1181         List<Instruction> instructions = new ArrayList<>();
1182         List<ActionInfo> actionsInfos = new ArrayList<>();
1183         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1184         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1185         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1186         instructions.add(writeInstruction);
1187         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1188
1189         // Install the flow entry in L3_LFIB_TABLE
1190         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1191
1192         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1193             10, flowRef, 0, 0,
1194             COOKIE_VM_LFIB_TABLE, matches, instructions);
1195
1196         mdsalManager.addFlow(confTx, dpId, flowEntity);
1197
1198         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1199     }
1200
1201     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId, Uint32 l3Vni,
1202         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1203         ProviderTypes extNwProvType) {
1204
1205         List<MatchInfo> mkMatches = new ArrayList<>();
1206
1207         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1208             dpnId, serviceId, customInstructions);
1209
1210         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1211             mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
1212         } else {
1213             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
1214         }
1215
1216         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1217             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1218                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1219                 String.format("%s:%s", "TST Flow Entry ", serviceId), 0, 0,
1220                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(serviceId.longValue()))),
1221                 mkMatches, customInstructions);
1222
1223         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1224     }
1225
1226     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(Uint32 routerId) {
1227         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1228             new RouterIdsKey(routerId)).build();
1229     }
1230
1231     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
1232         return NatConstants.SNAT_FLOWID_PREFIX + dpnId.toString() + NwConstants.FLOWID_SEPARATOR + tableId
1233                 + NwConstants.FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1234     }
1235
1236     @Override
1237     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1238         LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1239         String routerName = original.getRouterName();
1240         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1241         if (routerId == NatConstants.INVALID_ID) {
1242             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1243             return;
1244         }
1245         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1246             List<ListenableFuture<Void>> futures = new ArrayList<>();
1247             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1248                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1249                     Uint32 bgpVpnId = NatConstants.INVALID_ID;
1250                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1251                     if (bgpVpnUuid != null) {
1252                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1253                     }
1254                     //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1255                     /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1256                      * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1257                      */
1258                     Uint64 dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1259                     boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(Uint64
1260                             .valueOf(BigInteger.ZERO)));
1261                     Uuid networkId = original.getNetworkId();
1262                     // Check if its update on SNAT flag
1263                     boolean originalSNATEnabled = original.isEnableSnat();
1264                     boolean updatedSNATEnabled = update.isEnableSnat();
1265                     LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1266                             + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1267                         originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1268                     // Cluster Reboot Case Handling
1269                     // 1. DPN not elected during add event(due to none of the OVS connected)
1270                     // 2. Update event called with changes of parameters(but enableSnat is not changed)
1271                     // 3. First Elect dpnId and process other changes with valid dpnId
1272                     if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1273                         if (originalSNATEnabled && !updatedSNATEnabled) {
1274                             if (isPrimaryNaptSwitchNotSelected) {
1275                                 LOG.info("No Action to be taken when SNAT is disabled "
1276                                     + "with no Napt Switch Election for Router {}", routerName);
1277                                 return;
1278                             }
1279                             //SNAT disabled for the router
1280                             Uuid networkUuid = original.getNetworkId();
1281                             LOG.info("update : SNAT disabled for Router {}", routerName);
1282                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1283                             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1284                             handleDisableSnat(original, networkUuid, externalIps, false, vpnName,
1285                                     dpnId, routerId, removeFlowInvTx);
1286                         }  else if (updatedSNATEnabled) {
1287                             LOG.info("update : SNAT enabled for Router {}", routerName);
1288                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1289                                 true, writeFlowInvTx);
1290                             if (isPrimaryNaptSwitchNotSelected) {
1291                                 dpnId = selectNewNAPTSwitch(routerName);
1292                                 if (dpnId != null && !dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1293                                     handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1294                                 } else {
1295                                     LOG.error("update : Failed to elect Napt Switch During update event"
1296                                         + " of router {}", routerName);
1297                                 }
1298                             }
1299                         }
1300                         LOG.info("update : no need to process external/subnet changes as it's will taken care"
1301                             + "in handleDisableSnat/handleEnableSnat");
1302                         return;
1303                     }
1304                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1305                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1306                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1307                     }
1308
1309                     if (updatedSNATEnabled != originalSNATEnabled) {
1310                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1311                                 + "handleDisableSnat/handleEnableSnat");
1312                         return;
1313                     }
1314                     //Check if the Update is on External IPs
1315                     LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1316                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1317                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1318
1319                     //Check if the External IPs are removed during the update.
1320                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1321                     removedExternalIps.removeAll(updatedExternalIps);
1322                     if (removedExternalIps.size() > 0) {
1323                         LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1324                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1325                                 removedExternalIps, original.getExtGwMacAddress(),
1326                                 dpnId, networkId);
1327
1328                         for (String removedExternalIp : removedExternalIps) {
1329                 /*
1330                     1) Remove the mappings in the IntExt IP model which has external IP.
1331                     2) Remove the external IP in the ExternalCounter model.
1332                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1333                        least loaded external IP.
1334                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1335                     4) Increase the count of the allocated external IP by one.
1336                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1337                      i.e. the route for the external IP is absent.
1338                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1339                      the removed external IPs and also from the model.
1340                     7) Advertise to the BGP for removing the route for the removed external IPs.
1341                  */
1342
1343                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1344                             String externalIp = externalIpParts[0];
1345                             String externalIpPrefix = externalIpParts[1];
1346                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1347
1348                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1349                                     + "entries for removed external IP {}", externalIpAddrStr);
1350                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1351                             String vpnName = "";
1352                             if (vpnUuId != null) {
1353                                 vpnName = vpnUuId.getValue();
1354                             }
1355                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1356                                     update.getExtGwMacAddress(), removeFlowInvTx);
1357
1358                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1359                             //Get the internal IPs which are associated to the removed external IPs
1360                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1361                             List<String> removedInternalIps = new ArrayList<>();
1362                             for (IpMap ipMap : ipMaps) {
1363                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1364                                     removedInternalIps.add(ipMap.getInternalIp());
1365                                 }
1366                             }
1367
1368                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1369                             for (String removedInternalIp : removedInternalIps) {
1370                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1371                                                 + "router ID {} from the IntExtIP model",
1372                                         removedInternalIp, routerId);
1373                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1374                             }
1375
1376                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1377                                             + "router ID {} from the ExternalIpsCounter model.",
1378                                     externalIpAddrStr, routerId);
1379                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1380
1381                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1382                                     + "whose external IPs were removed.");
1383                             for (String removedInternalIp : removedInternalIps) {
1384                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1385                                         removedInternalIp, writeFlowInvTx);
1386                             }
1387                             LOG.debug("update : Remove the NAPT translation entries from "
1388                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1389                             //Get the internalIP and internal Port which were associated to the removed external IP.
1390                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1391                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1392                                     .builder(IntextIpPortMap.class)
1393                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1394                             Optional<IpPortMapping> ipPortMapping;
1395                             try {
1396                                 ipPortMapping = SingleTransactionDataBroker
1397                                             .syncReadOptional(dataBroker,
1398                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1399                             } catch (ReadFailedException e) {
1400                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1401                                 ipPortMapping = Optional.absent();
1402                             }
1403
1404                             if (ipPortMapping.isPresent()) {
1405                                 for (IntextIpProtocolType intextIpProtocolType :
1406                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1407                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1408                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1409                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1410                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1411                                             List<String> removedInternalIpPorts =
1412                                                     protoTypesIntIpPortsMap.get(protoType);
1413                                             if (removedInternalIpPorts != null) {
1414                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1415                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1416                                             } else {
1417                                                 removedInternalIpPorts = new ArrayList<>();
1418                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1419                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1420                                             }
1421                                         }
1422                                     }
1423                                 }
1424                             }
1425
1426                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1427                             // the removed external IP.
1428                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1429                                     protoTypesIntIpPortsMap.entrySet();
1430                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1431                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1432                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1433                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1434                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1435                                     // Remove the IP port map from the intext-ip-port-map model,
1436                                     // which were containing the removed external IP
1437                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1438                                             protocolType);
1439                                     //Remove the IP port incomint packer map.
1440                                     naptPacketInHandler.removeIncomingPacketMap(
1441                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1442                                     String[] removedInternalIpPortParts = removedInternalIpPort
1443                                             .split(NatConstants.COLON_SEPARATOR);
1444                                     if (removedInternalIpPortParts.length == 2) {
1445                                         String removedInternalIp = removedInternalIpPortParts[0];
1446                                         String removedInternalPort = removedInternalIpPortParts[1];
1447                                         List<String> removedInternalPortsList =
1448                                                 internalIpPortMap.get(removedInternalPort);
1449                                         if (removedInternalPortsList != null) {
1450                                             removedInternalPortsList.add(removedInternalPort);
1451                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1452                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1453                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1454                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1455                                             //Remove the NAPT translation entries from Outbound NAPT table
1456                                             naptEventHandler.removeNatFlows(dpnId,
1457                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1458                                                 routerId, removedInternalIp,
1459                                                 Integer.parseInt(removedInternalPort),
1460                                                 protocolType.getName());
1461                                             naptEventHandler.removeNatFlows(dpnId,
1462                                                 NwConstants.INBOUND_NAPT_TABLE,
1463                                                 routerId, removedInternalIp,
1464                                                 Integer.parseInt(removedInternalPort),
1465                                                 protocolType.getName());
1466                                         } else {
1467                                             removedInternalPortsList = new ArrayList<>();
1468                                             removedInternalPortsList.add(removedInternalPort);
1469                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1470                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1471                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1472                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1473                                             //Remove the NAPT translation entries from Outbound NAPT table
1474                                             naptEventHandler.removeNatFlows(dpnId,
1475                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1476                                                 routerId, removedInternalIp,
1477                                                 Integer.parseInt(removedInternalPort),
1478                                                 protocolType.getName());
1479                                             naptEventHandler.removeNatFlows(dpnId,
1480                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1481                                                 Integer.parseInt(removedInternalPort),
1482                                                 protocolType.getName());
1483                                         }
1484                                     }
1485                                 }
1486                             }
1487
1488                             // Delete the entry from SnatIntIpPortMap DS
1489                             Set<String> internalIps = internalIpPortMap.keySet();
1490                             for (String internalIp : internalIps) {
1491                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1492                                         + "model SnatIntIpPortMap", internalIp);
1493                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1494                             }
1495
1496                             naptManager.removeNaptPortPool(externalIp);
1497                         }
1498                         LOG.debug(
1499                                 "update : End processing of the External IPs removal for router {}", routerName);
1500                     }
1501
1502                     //Check if the External IPs are added during the update.
1503                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1504                     addedExternalIps.removeAll(originalExternalIps);
1505                     if (addedExternalIps.size() != 0) {
1506                         LOG.debug("update : Start processing of the External IPs addition for router {}",
1507                             routerName);
1508                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1509                                 update.getExtGwMacAddress(), dpnId,
1510                                 update.getNetworkId());
1511
1512                         for (String addedExternalIp : addedExternalIps) {
1513                 /*
1514                     1) Do nothing in the IntExtIp model.
1515                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1516                  */
1517                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1518                             String externalIp = externalIpParts[0];
1519                             String externalIpPrefix = externalIpParts[1];
1520                             String externalpStr = externalIp + "/" + externalIpPrefix;
1521                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1522                                             + "router ID {} in the ExternalIpsCounter model.",
1523                                     externalpStr, routerId);
1524                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1525                             subnetRegisterMapping(update, routerId);
1526                             LOG.info("update : Installing fib flow fo newly added Ips");
1527                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1528                         }
1529                         LOG.debug(
1530                                 "update : End processing of the External IPs addition during the update operation");
1531                     }
1532
1533                     //Check if its Update on subnets
1534                     LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1535                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1536                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1537                     Set<Uuid> addedSubnetIds =
1538                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1539                     if (originalSubnetIds != null) {
1540                         addedSubnetIds.removeAll(originalSubnetIds);
1541                     }
1542
1543                     //Check if the Subnet IDs are added during the update.
1544                     if (addedSubnetIds.size() != 0) {
1545                         LOG.debug(
1546                                 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1547                         for (Uuid addedSubnetId : addedSubnetIds) {
1548                 /*
1549                     1) Select the least loaded external IP for the subnet and store the mapping of the
1550                     subnet IP and the external IP in the IntExtIp model.
1551                     2) Increase the count of the selected external IP by one.
1552                     3) Advertise to the BGP if external IP is allocated for the first time for the
1553                     router i.e. the route for the external IP is absent.
1554                  */
1555                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1556                             if (subnetIp != null) {
1557                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1558                                         writeFlowInvTx);
1559                             }
1560                         }
1561                         LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1562                     }
1563
1564                     //Check if the Subnet IDs are removed during the update.
1565                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1566                     removedSubnetIds.removeAll(updatedSubnetIds);
1567                     if (removedSubnetIds.size() != 0) {
1568                         LOG.debug(
1569                                 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1570                         for (Uuid removedSubnetId : removedSubnetIds) {
1571                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1572                             if (subnetAddr != null) {
1573                     /*
1574                         1) Remove the subnet IP and the external IP in the IntExtIp map
1575                         2) Decrease the count of the coresponding external IP by one.
1576                         3) Advertise to the BGP for removing the routes of the corresponding external
1577                         IP if its not allocated to any other internal IP.
1578                     */
1579
1580                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1581                                         subnetAddr[0] + "/" + subnetAddr[1]);
1582                                 if (externalIp == null) {
1583                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1584                                             routerId, subnetAddr[0]);
1585                                     return;
1586                                 }
1587
1588                                 naptManager.updateCounter(routerId, externalIp, false);
1589                                 // Traverse entire model of external-ip counter whether external ip is not
1590                                 // used by any other internal ip in any router
1591                                 if (!isExternalIpAllocated(externalIp)) {
1592                                     LOG.debug("update : external ip is not allocated to any other "
1593                                             + "internal IP so proceeding to remove routes");
1594                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1595                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1596                                             removeFlowInvTx);
1597                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1598                                                     + "router {} with networkId {} and externalIp {}",
1599                                             dpnId, routerId, networkId, externalIp);
1600                                 }
1601
1602                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1603                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1604                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1605                             }
1606                         }
1607                         LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1608                     }
1609                 }));
1610             }));
1611             return futures;
1612         }, NatConstants.NAT_DJC_MAX_RETRIES);
1613     }
1614
1615     private boolean isExternalIpAllocated(String externalIp) {
1616         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1617         Optional<ExternalIpsCounter> externalCountersData;
1618         try {
1619             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1620                         LogicalDatastoreType.OPERATIONAL, id);
1621         } catch (ReadFailedException e) {
1622             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1623             externalCountersData = Optional.absent();
1624         }
1625         if (externalCountersData.isPresent()) {
1626             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1627             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1628                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1629                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1630                         if (externalIpCount.getCounter().toJava() != 0) {
1631                             return true;
1632                         }
1633                         break;
1634                     }
1635                 }
1636             }
1637         }
1638         return false;
1639     }
1640
1641     private void allocateExternalIp(Uint64 dpnId, Routers router, Uint32 routerId, String routerName,
1642             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1643         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1644         try {
1645             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1646             if (address instanceof Inet6Address) {
1647                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1648                 return;
1649             }
1650         } catch (UnknownHostException e) {
1651             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1652             return;
1653         }
1654         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1655         if (leastLoadedExtIpAddr != null) {
1656             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1657             String leastLoadedExtIp = externalIpParts[0];
1658             String leastLoadedExtIpPrefix = externalIpParts[1];
1659             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1660             subnetIp = subnetIpParts[0];
1661             String subnetIpPrefix = subnetIpParts[1];
1662             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1663             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1664                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1665                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1666             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1667
1668
1669             // Check if external IP is already assigned a route. (i.e. External IP is previously
1670             // allocated to any of the subnets)
1671             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1672             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1673             Uint32 label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1674             if (label != null) {
1675                 //update
1676                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1677                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1678                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1679                     label, internalIp, leastLoadedExtIpAddrStr);
1680                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1681                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1682                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1683                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1684                 return;
1685             }
1686
1687             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1688             // for the first time and hence not having a route.
1689             //Get the VPN Name using the network ID
1690             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1691             if (vpnName != null) {
1692                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1693                 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1694                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1695                         + "added after gateway-set");
1696                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1697                     if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1698                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1699                         return;
1700                     }
1701                 }
1702                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1703                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1704                     writeFlowInvTx);
1705             }
1706         }
1707     }
1708
1709     @Nullable
1710     protected Uint32 checkExternalIpLabel(Uint32 routerId, String externalIp) {
1711         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1712         for (IpMap ipMap : ipMaps) {
1713             if (ipMap.getExternalIp().equals(externalIp)) {
1714                 if (ipMap.getLabel() != null) {
1715                     return ipMap.getLabel();
1716                 }
1717             }
1718         }
1719         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1720         return null;
1721     }
1722
1723     @Override
1724     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1725         LOG.trace("remove : Router delete method");
1726         /*
1727         ROUTER DELETE SCENARIO
1728         1) Get the router ID from the event.
1729         2) Build the cookie information from the router ID.
1730         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1731         4) Build the flow with the cookie value.
1732         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1733         6) Remove the flows from the other switches which points to the primary and secondary
1734          switches for the flows related the router ID.
1735         7) Get the list of external IP address maintained for the router ID.
1736         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1737         9) Withdraw the corresponding routes from the BGP.
1738          */
1739
1740         if (identifier == null || router == null) {
1741             LOG.error("remove : returning without processing since routers is null");
1742             return;
1743         }
1744
1745         String routerName = router.getRouterName();
1746         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1747             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1748                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1749                         routerName);
1750                 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1751                 if (routerId == NatConstants.INVALID_ID) {
1752                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1753                             routerName);
1754                     return;
1755                 }
1756                 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1757                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1758                 if (bgpVpnUuid != null) {
1759                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1760                 }
1761                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1762                         tx);
1763                 Uuid networkUuid = router.getNetworkId();
1764
1765                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1766                 if (primarySwitchId == null || primarySwitchId.equals(Uint64.ZERO)) {
1767                     // No NAPT switch for external router, probably because the router is not attached to
1768                     // any
1769                     // internal networks
1770                     LOG.debug(
1771                             "No NAPT switch for router {}, check if router is attached to any internal "
1772                                     + "network",
1773                             routerName);
1774                     return;
1775                 } else {
1776                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1777                     final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
1778                     handleDisableSnat(router, networkUuid, externalIps, true, vpnName, primarySwitchId,
1779                             routerId, tx);
1780                 }
1781                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1782                     == NatConstants.INVALID_ID) {
1783                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1784                 }
1785             })), NatConstants.NAT_DJC_MAX_RETRIES);
1786     }
1787
1788     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1789                                   boolean routerFlag, @Nullable String vpnName, Uint64 naptSwitchDpnId,
1790                                   Uint32 routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1791         LOG.info("handleDisableSnat : Entry");
1792         String routerName = router.getRouterName();
1793         try {
1794             if (routerFlag) {
1795                 removeNaptSwitch(routerName);
1796             } else {
1797                 updateNaptSwitch(routerName, Uint64.valueOf(BigInteger.ZERO));
1798             }
1799
1800             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1801             naptManager.removeExternalCounter(routerId);
1802
1803             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1804             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1805                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1806                     + "router ID {} from RouterNaptSwitch model", routerId);
1807                 return;
1808             }
1809             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1810                     networkUuid);
1811             if (extNwProvType == null) {
1812                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1813                 return;
1814             }
1815             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1816             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1817                     externalSubnetList, removeFlowInvTx, extNwProvType);
1818             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1819             String externalSubnetVpn = null;
1820             for (Uuid externalSubnetId : externalSubnetList) {
1821                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1822                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1823                 if (externalSubnet.isPresent()) {
1824                     externalSubnetVpn =  externalSubnetId.getValue();
1825                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1826                             router.getExtGwMacAddress(), removeFlowInvTx);
1827                 }
1828             }
1829             if (externalSubnetVpn == null) {
1830                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1831                         router.getExtGwMacAddress(), removeFlowInvTx);
1832             }
1833             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1834             // for the router ID.
1835             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1836                     + "router ID {} in the DS", routerId);
1837             naptManager.removeMapping(routerId);
1838         } catch (InterruptedException | ExecutionException e) {
1839             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1840         }
1841         LOG.info("handleDisableSnat : Exit");
1842     }
1843
1844     // TODO Clean up the exception handling
1845     @SuppressWarnings("checkstyle:IllegalCatch")
1846     public void handleDisableSnatInternetVpn(String routerName, Uint32 routerId, Uuid networkUuid,
1847                                              @NonNull Collection<String> externalIps,
1848                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1849         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1850                 + "with internet vpn {}", routerName, vpnId);
1851         try {
1852             Uint64 naptSwitchDpnId = null;
1853             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1854                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1855             Optional<RouterToNaptSwitch> rtrToNapt;
1856             try {
1857                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1858                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1859             } catch (ReadFailedException e) {
1860                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1861                 rtrToNapt = Optional.absent();
1862             }
1863             if (rtrToNapt.isPresent()) {
1864                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1865             }
1866             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1867
1868             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1869                     writeFlowInvTx);
1870             try {
1871                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1872                 if (extGwMacAddress != null) {
1873                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1874                             + "External Router ID {}", extGwMacAddress, routerId);
1875                 } else {
1876                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1877                             + "External Router ID {}", routerId);
1878                     return;
1879                 }
1880                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1881                         writeFlowInvTx);
1882             } catch (Exception ex) {
1883                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1884                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1885             }
1886             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1887                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1888             }
1889         } catch (InterruptedException | ExecutionException e) {
1890             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1891                     + "with internet vpn {}", routerName, vpnId, e);
1892         }
1893         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1894                 routerName, vpnId);
1895     }
1896
1897     // TODO Clean up the exception handling
1898     @SuppressWarnings("checkstyle:IllegalCatch")
1899     public void updateNaptSwitch(String routerName, Uint64 naptSwitchId) {
1900         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1901             .setPrimarySwitchId(naptSwitchId).build();
1902         try {
1903             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1904                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1905         } catch (Exception ex) {
1906             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1907                 naptSwitchId, routerName);
1908         }
1909         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1910             naptSwitchId, routerName);
1911     }
1912
1913     protected void removeNaptSwitch(String routerName) {
1914         // Remove router and switch from model
1915         InstanceIdentifier<RouterToNaptSwitch> id =
1916             InstanceIdentifier.builder(NaptSwitches.class)
1917                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1918         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1919         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1920         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1921         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1922     }
1923
1924     public void removeNaptFlowsFromActiveSwitch(Uint32 routerId, String routerName,
1925                                                 Uint64 dpnId, Uuid networkId, String vpnName,
1926                                                 @NonNull Collection<String> externalIps,
1927                                                 Collection<Uuid> externalSubnetList,
1928                                                 TypedReadWriteTransaction<Configuration> confTx,
1929                                                 ProviderTypes extNwProvType)
1930             throws InterruptedException, ExecutionException {
1931
1932         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1933         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1934
1935         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1936         // traffic which comes from the  VMs of the NAPT switches)
1937         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1938         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1939
1940         LOG.info(
1941             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1942                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1943         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1944
1945         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1946         // traffic which comes from the VMs of the non NAPT switches)
1947         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1948                 elanManager, idManager, routerId, routerName);
1949         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
1950         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1951         LOG.info(
1952             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1953                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1954         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1955
1956         //Remove the flow table 25->44 from NAPT Switch
1957         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1958             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1959         }
1960
1961         //Remove the Outbound flow entry which forwards the packet to FIB Table
1962         LOG.info(
1963             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1964                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1965
1966         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1967             NwConstants.IP_PROT_TCP);
1968         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1969             outboundTcpNatFlowRef);
1970         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1971
1972         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1973             NwConstants.IP_PROT_UDP);
1974         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1975             outboundUdpNatFlowRef);
1976         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1977
1978         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1979             NwConstants.IP_PROT_ICMP);
1980         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1981             icmpDropFlowRef);
1982         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1983         boolean lastRouterOnExternalNetwork =
1984             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1985         if (lastRouterOnExternalNetwork) {
1986             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1987         }
1988         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1989         // External Subnet Vpn Id.
1990         for (Uuid externalSubnetId : externalSubnetList) {
1991             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1992             if (subnetVpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1993                 dataBroker, externalSubnetId, routerName, dpnId)) {
1994                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1995                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1996                     natPfibSubnetFlowRef);
1997                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1998                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1999                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
2000                     subnetVpnId, dpnId);
2001             }
2002         }
2003
2004         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
2005         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
2006         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
2007
2008         LOG.info(
2009             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2010             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2011         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2012
2013         if (lastRouterOnExternalNetwork) {
2014             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2015             // - This does not work since ext-routers is deleted already - no network info
2016             //Get the VPN ID from the ExternalNetworks model
2017             Uint32 vpnId = NatConstants.INVALID_ID;
2018             if (vpnName == null || vpnName.isEmpty()) {
2019                 // ie called from router delete cases
2020                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2021                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2022                 if (vpnUuid != null) {
2023                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2024                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2025                               + "or disableSNAT scenario", vpnId, networkId);
2026                 }
2027             } else {
2028                 // ie called from disassociate vpn case
2029                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2030                     vpnName);
2031                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2032                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2033                     vpnId);
2034             }
2035
2036             if (vpnId != NatConstants.INVALID_ID) {
2037                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2038                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2039                 FlowEntity natPfibVpnFlowEntity =
2040                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2041                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2042                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2043                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2044             }
2045         }
2046
2047         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2048         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2049         if (ipPortMapping == null) {
2050             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2051             return;
2052         }
2053
2054         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2055             String protocol = intextIpProtocolType.getProtocol().name();
2056             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2057                 String ipPortInternal = ipPortMap.getIpPortInternal();
2058                 String[] ipPortParts = ipPortInternal.split(":");
2059                 if (ipPortParts.length != 2) {
2060                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2061                     return;
2062                 }
2063                 String internalIp = ipPortParts[0];
2064                 String internalPort = ipPortParts[1];
2065
2066                 //Build the flow for the outbound NAPT table
2067                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2068                     + NatConstants.COLON_SEPARATOR + internalPort);
2069                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2070                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2071                 FlowEntity outboundNaptFlowEntity =
2072                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2073
2074                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2075                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2076                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2077
2078                  //Build the flow for the inbound NAPT table
2079                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2080                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2081                 FlowEntity inboundNaptFlowEntity =
2082                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2083
2084                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2085                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2086                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2087             }
2088         }
2089     }
2090
2091     protected void removeNaptFibExternalOutputFlows(Uint32 routerId, Uint64 dpnId, Uuid networkId,
2092                                                     @NonNull Collection<String> externalIps,
2093                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2094             throws ExecutionException, InterruptedException {
2095         Uint32 extVpnId = NatConstants.INVALID_ID;
2096         if (networkId != null) {
2097             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2098             if (vpnUuid != null) {
2099                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2100             } else {
2101                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2102             }
2103         } else {
2104             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2105             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2106         }
2107         if (extVpnId == NatConstants.INVALID_ID) {
2108             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2109             extVpnId = routerId;
2110         }
2111         for (String ip : externalIps) {
2112             String extIp = removeMaskFromIp(ip);
2113             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2114             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2115                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2116                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2117             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2118             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2119         }
2120     }
2121
2122     private String removeMaskFromIp(String ip) {
2123         if (ip != null && !ip.trim().isEmpty()) {
2124             return ip.split("/")[0];
2125         }
2126         return ip;
2127     }
2128
2129     public void removeNaptFlowsFromActiveSwitchInternetVpn(Uint32 routerId, String routerName,
2130                                                            Uint64 dpnId, Uuid networkId, String vpnName,
2131                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2132             throws ExecutionException, InterruptedException {
2133         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2134         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2135
2136         //Remove the NAPT PFIB TABLE entry
2137         Uint32 vpnId = NatConstants.INVALID_ID;
2138         if (vpnName != null) {
2139             // ie called from disassociate vpn case
2140             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2141                     + "with vpnName {}", vpnName);
2142             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2143             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2144                     vpnId);
2145         }
2146
2147         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2148                 networkId, routerName, dpnId)) {
2149             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2150             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2151             FlowEntity natPfibVpnFlowEntity =
2152                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2153             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2154                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2155             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2156
2157             // Remove IP-PORT active NAPT entries and release port from IdManager
2158             // For the router ID get the internal IP , internal port and the corresponding
2159             // external IP and external Port.
2160             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2161             if (ipPortMapping == null) {
2162                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2163                 return;
2164             }
2165             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2166                 String protocol = intextIpProtocolType.getProtocol().name();
2167                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2168                     String ipPortInternal = ipPortMap.getIpPortInternal();
2169                     String[] ipPortParts = ipPortInternal.split(":");
2170                     if (ipPortParts.length != 2) {
2171                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2172                                 + "and port");
2173                         return;
2174                     }
2175                     String internalIp = ipPortParts[0];
2176                     String internalPort = ipPortParts[1];
2177
2178                     //Build the flow for the outbound NAPT table
2179                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2180                             + NatConstants.COLON_SEPARATOR + internalPort);
2181                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2182                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2183                     FlowEntity outboundNaptFlowEntity =
2184                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2185
2186                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2187                             + "active switch with the DPN ID {} and router ID {}",
2188                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2189                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2190
2191                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2192                     final String externalIp = ipPortExternal.getIpAddress();
2193
2194                     //Build the flow for the inbound NAPT table
2195                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2196                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2197                     FlowEntity inboundNaptFlowEntity =
2198                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2199
2200                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2201                             + "active active switch with the DPN ID {} and router ID {}",
2202                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2203                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2204
2205                     // Finally release port from idmanager
2206                     String internalIpPort = internalIp + ":" + internalPort;
2207                     naptManager.removePortFromPool(internalIpPort, externalIp);
2208
2209                     //Remove sessions from models
2210                     naptManager.removeIpPortMappingForRouterID(routerId);
2211                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2212                 }
2213             }
2214         } else {
2215             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2216         }
2217     }
2218
2219     public void removeFlowsFromNonActiveSwitches(Uint32 routerId, String routerName,
2220             Uint64 naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2221             throws ExecutionException, InterruptedException {
2222         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2223
2224         // Remove the flows from the other switches which points to the primary and secondary switches
2225         // for the flows related the router ID.
2226         List<Uint64> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2227         if (allSwitchList.isEmpty()) {
2228             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2229             return;
2230         }
2231         for (Uint64 dpnId : allSwitchList) {
2232             if (!naptSwitchDpnId.equals(dpnId)) {
2233                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2234
2235                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2236                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2237                 FlowEntity preSnatFlowEntity =
2238                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2239
2240                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2241                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2242                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2243
2244                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2245                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2246                     NatUtil.getGroupIdKey(routerName));
2247                 if (groupId != NatConstants.INVALID_ID) {
2248                     LOG.info(
2249                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2250                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2251                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId.longValue());
2252                 } else {
2253                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2254                 }
2255             }
2256         }
2257     }
2258
2259     public void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, @Nullable Uuid networkUuid,
2260                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2261                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2262             throws ExecutionException, InterruptedException {
2263         //Withdraw the corresponding routes from the BGP.
2264         //Get the network ID using the router ID.
2265         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2266                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2267         if (networkUuid == null) {
2268             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2269             return;
2270         }
2271
2272         if (externalIps.isEmpty()) {
2273             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2274             return;
2275         }
2276
2277         if (vpnName == null) {
2278             //Get the VPN Name using the network ID
2279             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2280             if (vpnName == null) {
2281                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2282                     networkUuid, routerId);
2283                 return;
2284             }
2285         }
2286         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2287
2288         //Remove custom FIB routes
2289         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2290         for (String extIp : externalIps) {
2291             extIp = NatUtil.validateAndAddNetworkMask(extIp);
2292             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2293         }
2294     }
2295
2296     protected void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2297                                             final Uuid networkUuid, String extGwMacAddress,
2298                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2299             throws ExecutionException, InterruptedException {
2300         clearBgpRoutes(extIp, vpnName);
2301         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2302                 removeFlowInvTx);
2303     }
2304
2305     protected void delFibTsAndReverseTraffic(final Uint64 dpnId, String routerName, Uint32 routerId, String extIp,
2306                                              String vpnName, Uuid extNetworkId, Uint32 tempLabel,
2307                                              String gwMacAddress, boolean switchOver,
2308                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2309             throws ExecutionException, InterruptedException {
2310         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2311         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2312         if (routerName == null) {
2313             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2314             return;
2315         }
2316         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2317         if (extNwProvType == null) {
2318             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2319             return;
2320         }
2321         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2322          * external network provided type is VxLAN
2323          */
2324         if (extNwProvType == ProviderTypes.VXLAN) {
2325             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2326             );
2327             return;
2328         }
2329         if (tempLabel.longValue() < 0) {
2330             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2331                     extIp, routerId);
2332             return;
2333         }
2334         final Uint32 label = tempLabel;
2335         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2336         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2337             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2338             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2339             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2340                     extRouter);
2341
2342             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2343                     externalSubnetId);
2344
2345             if (externalSubnet.isPresent()) {
2346                 vpnName =  externalSubnetId.getValue();
2347             }
2348         }
2349         final String externalVpn = vpnName;
2350         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2351                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2352                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2353         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2354
2355         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2356         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2357         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2358             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2359             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2360         }
2361         if (!switchOver) {
2362             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2363                 Futures.transformAsync(future, result -> {
2364                     //Release label
2365                     if (result.isSuccessful()) {
2366                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2367                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2368                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2369                         return vpnService.removeVpnLabel(labelInput);
2370                     } else {
2371                         String errMsg =
2372                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2373                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2374                         LOG.error(errMsg);
2375                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2376                     }
2377                 }, MoreExecutors.directExecutor());
2378
2379             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2380
2381                 @Override
2382                 public void onFailure(@NonNull Throwable error) {
2383                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2384                         + "got external ip {}", label, extIp, error);
2385                 }
2386
2387                 @Override
2388                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2389                     if (result.isSuccessful()) {
2390                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2391                             + "from VPN {}", externalIp, externalVpn);
2392                     } else {
2393                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2394                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2395                     }
2396                 }
2397             }, MoreExecutors.directExecutor());
2398         } else {
2399             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2400                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2401         }
2402     }
2403
2404     private void delFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2405                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2406                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2407             throws ExecutionException, InterruptedException {
2408         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2409         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2410         if (routerName == null) {
2411             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2412             return;
2413         }
2414         //Get the external network provider type from networkId
2415         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2416         if (extNwProvType == null) {
2417             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2418                     networkUuid);
2419             return;
2420         }
2421         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2422          *  external network provided type is VxLAN
2423          */
2424         if (extNwProvType == ProviderTypes.VXLAN) {
2425             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2426             return;
2427         }
2428         //Get IPMaps from the DB for the router ID
2429         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2430         if (dbIpMaps.isEmpty()) {
2431             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2432             return;
2433         }
2434
2435         Uint32 tempLabel = NatConstants.INVALID_ID;
2436         for (IpMap dbIpMap : dbIpMaps) {
2437             String dbExternalIp = dbIpMap.getExternalIp();
2438             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2439             //Select the IPMap, whose external IP is the IP for which FIB is installed
2440             if (extIp.equals(dbExternalIp)) {
2441                 tempLabel = dbIpMap.getLabel();
2442                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2443                     tempLabel, dbExternalIp, routerId);
2444                 break;
2445             }
2446         }
2447         if (tempLabel == NatConstants.INVALID_ID) {
2448             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2449                     extIp, routerId);
2450             return;
2451         }
2452
2453         final Uint32 label = tempLabel;
2454         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2455         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2456                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2457                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2458         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2459
2460         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2461         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2462         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2463             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2464             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2465         }
2466         if (!switchOver) {
2467             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2468                     Futures.transformAsync(future, result -> {
2469                         //Release label
2470                         if (result.isSuccessful()) {
2471                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2472                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2473                             return vpnService.removeVpnLabel(labelInput);
2474                         } else {
2475                             String errMsg =
2476                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2477                                             + "prefix %s Failed - %s",
2478                                             dpnId, externalIp, result.getErrors());
2479                             LOG.error(errMsg);
2480                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2481                         }
2482                     }, MoreExecutors.directExecutor());
2483
2484             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2485
2486                 @Override
2487                 public void onFailure(@NonNull Throwable error) {
2488                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2489                 }
2490
2491                 @Override
2492                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2493                     if (result.isSuccessful()) {
2494                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2495                             + "from VPN {}", externalIp, vpnName);
2496                     } else {
2497                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2498                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2499                     }
2500                 }
2501             }, MoreExecutors.directExecutor());
2502         } else {
2503             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2504                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2505         }
2506     }
2507
2508     protected void clearFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, Uuid networkUuid,
2509             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2510             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2511         //Withdraw the corresponding routes from the BGP.
2512         //Get the network ID using the router ID.
2513         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2514                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2515         if (networkUuid == null) {
2516             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2517             return;
2518         }
2519
2520         if (externalIps == null || externalIps.isEmpty()) {
2521             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2522             return;
2523         }
2524
2525         if (vpnName == null) {
2526             //Get the VPN Name using the network ID
2527             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2528             if (vpnName == null) {
2529                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2530                     networkUuid, routerId);
2531                 return;
2532             }
2533         }
2534         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2535
2536         //Remove custom FIB routes
2537         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2538         for (String extIp : externalIps) {
2539             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2540                     writeFlowInvTx);
2541         }
2542     }
2543
2544     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2545         //Inform BGP about the route removal
2546         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2547         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2548         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName);
2549     }
2550
2551     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
2552             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2553         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2554         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2555             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2556         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2557     }
2558
2559     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
2560             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2561         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2562         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2563         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2564         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2565     }
2566
2567     /**
2568      * router association to vpn.
2569      *
2570      * @param routerName - Name of router
2571      * @param routerId - router id
2572      * @param bgpVpnName BGP VPN name
2573      */
2574     public void changeLocalVpnIdToBgpVpnId(String routerName, Uint32 routerId, String extNetwork, String bgpVpnName,
2575             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2576         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2577         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2578             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2579
2580             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2581
2582             if (bgpVpnId != NatConstants.INVALID_ID) {
2583                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2584                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2585                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2586                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2587                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2588                     getRoutersIdentifier(bgpVpnId), rtrs);
2589
2590                 // Get the allocated Primary NAPT Switch for this router
2591                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2592                         routerId, bgpVpnId);
2593                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2594
2595                 // Get the group ID
2596                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2597                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, new Uuid(extNetwork),
2598                     true, writeFlowInvTx, extNwProvType);
2599             }
2600         }
2601     }
2602
2603     /**
2604      * router disassociation from vpn.
2605      *
2606      * @param routerName - Name of router
2607      * @param routerId - router id
2608      * @param bgpVpnName BGP VPN name
2609      */
2610     public void changeBgpVpnIdToLocalVpnId(String routerName, Uint32 routerId, String bgpVpnName, String extNetwork,
2611             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2612         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2613         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2614             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2615             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2616
2617             // Get the allocated Primary NAPT Switch for this router
2618             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2619
2620             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2621             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2622
2623             // Get the group ID
2624             Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2625             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId,
2626                 new Uuid(extNetwork), true, writeFlowInvTx, extNwProvType);
2627         }
2628     }
2629
2630     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2631         InstanceIdentifier<Routers> routerInstanceIndentifier =
2632             InstanceIdentifier.builder(ExtRouters.class)
2633                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2634         try {
2635             Optional<Routers> routerData = SingleTransactionDataBroker
2636                     .syncReadOptional(dataBroker,
2637                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2638             return routerData.isPresent() && routerData.get().isEnableSnat();
2639         } catch (ReadFailedException e) {
2640             LOG.error("Failed to read data for router id {}", routerUuid, e);
2641             return false;
2642         }
2643     }
2644
2645     public void installFlowsWithUpdatedVpnId(Uint64 primarySwitchId, String routerName, Uint32 bgpVpnId,
2646                                              Uint32 routerId, Uuid extNwUuid, boolean isSnatCfgd,
2647                                              TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2648
2649         Uint32 changedVpnId = bgpVpnId;
2650         String idType = "BGP VPN";
2651         if (bgpVpnId == NatConstants.INVALID_ID) {
2652             changedVpnId = routerId;
2653             idType = "router";
2654         }
2655
2656         List<Uint64> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2657         if (switches.isEmpty()) {
2658             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2659             return;
2660         }
2661         for (Uint64 dpnId : switches) {
2662             // Update the BGP VPN ID in the SNAT miss entry to group
2663             if (!dpnId.equals(primarySwitchId)) {
2664                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2665                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2666                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2667                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2668                     NatUtil.getGroupIdKey(routerName));
2669                 if (groupId != NatConstants.INVALID_ID) {
2670                     if (!isSnatCfgd) {
2671                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2672                     }
2673                     LOG.debug(
2674                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2675                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2676                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2677                         groupId, changedVpnId);
2678                     mdsalManager.addFlow(confTx, flowEntity);
2679                 } else {
2680                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2681                 }
2682             } else {
2683                 LOG.debug(
2684                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2685                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2686                 FlowEntity flowEntity =
2687                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2688                 mdsalManager.addFlow(confTx, flowEntity);
2689
2690                 LOG.debug(
2691                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2692                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2693                         idType, changedVpnId, primarySwitchId);
2694                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2695                         changedVpnId, confTx, extNwProvType);
2696
2697                 LOG.debug(
2698                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2699                                 + "which punts the packet to the controller in the Primary switch {}",
2700                         idType, changedVpnId, primarySwitchId);
2701                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2702
2703                 LOG.debug(
2704                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2705                                 + " outgoing packet to FIB Table in the Primary switch {}",
2706                         idType, changedVpnId, primarySwitchId);
2707                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2708
2709                 LOG.debug(
2710                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2711                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2712                                 + " {}", idType, changedVpnId, primarySwitchId);
2713                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2714
2715                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2716                         primarySwitchId);
2717                 //Get the VPN ID from the ExternalNetworks model
2718                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, extNwUuid);
2719                 if (vpnUuid != null) {
2720                     Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2721                     //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table
2722                     // matching on the VPN ID.
2723                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
2724                         installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2725                     }
2726                 } else {
2727                     LOG.error("NAT Service : vpnUuid is null");
2728                 }
2729             }
2730         }
2731     }
2732
2733     public void updateNaptFlowsWithVpnId(Uint64 dpnId, String routerName, Uint32 routerId, Uint32 bgpVpnId) {
2734         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2735         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2736         if (ipPortMapping == null) {
2737             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2738             return;
2739         }
2740         // Get the External Gateway MAC Address
2741         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2742         if (extGwMacAddress != null) {
2743             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2744                     extGwMacAddress, routerId);
2745         } else {
2746             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2747                     routerId);
2748             return;
2749         }
2750         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2751             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2752                 String ipPortInternal = ipPortMap.getIpPortInternal();
2753                 String[] ipPortParts = ipPortInternal.split(":");
2754                 if (ipPortParts.length != 2) {
2755                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2756                     return;
2757                 }
2758                 String internalIp = ipPortParts[0];
2759                 String internalPort = ipPortParts[1];
2760                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2761                         internalIp, internalPort);
2762                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2763                 NAPTEntryEvent.Protocol protocol;
2764                 switch (protocolTypes) {
2765                     case TCP:
2766                         protocol = NAPTEntryEvent.Protocol.TCP;
2767                         break;
2768                     case UDP:
2769                         protocol = NAPTEntryEvent.Protocol.UDP;
2770                         break;
2771                     default:
2772                         protocol = NAPTEntryEvent.Protocol.TCP;
2773                 }
2774                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2775                 SessionAddress externalAddress =
2776                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2777                 Uint32 internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2778                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2779                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2780                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2781                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2782             }
2783         }
2784     }
2785
2786     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName, Uint32 groupId,
2787                                                           Uint32 changedVpnId) {
2788
2789         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2790             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2791         List<MatchInfo> matches = new ArrayList<>();
2792         matches.add(MatchEthernetType.IPV4);
2793         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2794                 MetaDataUtil.METADATA_MASK_VRFID));
2795
2796         List<ActionInfo> actionsInfo = new ArrayList<>();
2797         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2798                 elanManager, idManager, changedVpnId, routerName);
2799         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
2800         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2801                 actionsInfo);
2802         actionsInfo.add(new ActionGroup(groupId.longValue()));
2803         List<InstructionInfo> instructions = new ArrayList<>();
2804         instructions.add(new InstructionApplyActions(actionsInfo));
2805         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2806         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2807             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2808             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2809
2810         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2811         return flowEntity;
2812     }
2813
2814     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(Uint64 dpId, String routerName,
2815                                                                         Uint32 changedVpnId) {
2816
2817         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2818                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2819         List<MatchInfo> matches = new ArrayList<>();
2820         matches.add(MatchEthernetType.IPV4);
2821         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2822                 MetaDataUtil.METADATA_MASK_VRFID));
2823
2824         List<InstructionInfo> instructions = new ArrayList<>();
2825         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2826
2827         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2828         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2829             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2830             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2831
2832         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2833         return flowEntity;
2834     }
2835
2836     // TODO : Replace this with ITM Rpc once its available with full functionality
2837     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(Uint64 dpnId, String routerName,
2838                                                                      Uint32 routerId, Uint32 changedVpnId,
2839                                                                      TypedWriteTransaction<Configuration> confTx,
2840                                                                      ProviderTypes extNwProvType) {
2841
2842         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2843             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2844         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2845                 extNwProvType);
2846         mdsalManager.addFlow(confTx, flowEntity);
2847     }
2848
2849     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName,
2850                                                          Uint32 routerIdLongVal, Uint32 changedVpnId,
2851                                                          ProviderTypes extNwProvType) {
2852         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2853             dpId, routerName, changedVpnId);
2854         List<MatchInfo> matches = new ArrayList<>();
2855         matches.add(MatchEthernetType.IPV4);
2856
2857         Uint64 tunnelId = Uint64.valueOf(changedVpnId);
2858         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2859             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2860         }
2861         matches.add(new MatchTunnelId(tunnelId));
2862
2863         List<InstructionInfo> instructions = new ArrayList<>();
2864         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2865             MetaDataUtil.METADATA_MASK_VRFID));
2866         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2867         Uint32 routerId = routerIdLongVal;
2868         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
2869         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2870             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2871             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2872         return flowEntity;
2873     }
2874
2875     public void createOutboundTblEntryWithBgpVpn(Uint64 dpnId, Uint32 routerId, Uint32 changedVpnId,
2876                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2877         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2878             dpnId, routerId, changedVpnId);
2879         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2880                 NwConstants.IP_PROT_TCP);
2881         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2882         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2883
2884         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2885                 NwConstants.IP_PROT_UDP);
2886         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2887         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2888
2889         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2890         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2891         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2892     }
2893
2894     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(Uint64 dpId, Uint32 routerId,
2895                                                            Uint32 changedVpnId, int protocol) {
2896         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2897             dpId, routerId, changedVpnId);
2898         Uint64 cookie = getCookieOutboundFlow(routerId);
2899         List<MatchInfo> matches = new ArrayList<>();
2900         matches.add(MatchEthernetType.IPV4);
2901         matches.add(new MatchIpProtocol((short)protocol));
2902         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2903                 MetaDataUtil.METADATA_MASK_VRFID));
2904
2905         List<InstructionInfo> instructions = new ArrayList<>();
2906         List<ActionInfo> actionsInfos = new ArrayList<>();
2907         actionsInfos.add(new ActionPuntToController());
2908         if (snatPuntTimeout != 0) {
2909             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2910         }
2911         instructions.add(new InstructionApplyActions(actionsInfos));
2912
2913         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2914         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2915             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2916         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2917         return flowEntity;
2918     }
2919
2920     public void installNaptPfibEntryWithBgpVpn(Uint64 dpnId, Uint32 segmentId, Uint32 changedVpnId,
2921                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2922         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2923             dpnId, segmentId, changedVpnId);
2924         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2925         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2926     }
2927
2928     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(Uint64 dpId, Uint32 segmentId, Uint32 changedVpnId) {
2929
2930         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2931             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2932         List<MatchInfo> matches = new ArrayList<>();
2933         matches.add(MatchEthernetType.IPV4);
2934         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2935                 MetaDataUtil.METADATA_MASK_VRFID));
2936
2937         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2938         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2939         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
2940         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2941         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2942
2943         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2944         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2945             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2946             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2947         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2948         return flowEntity;
2949     }
2950
2951     @Override
2952     protected ExternalRoutersListener getDataTreeChangeListener() {
2953         return ExternalRoutersListener.this;
2954     }
2955
2956     protected void installNaptPfibEntriesForExternalSubnets(String routerName, Uint64 dpnId,
2957                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2958         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2959                 routerName);
2960         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2961             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2962             if (subnetVpnId != NatConstants.INVALID_ID) {
2963                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2964                     + "and vpnId {}", dpnId, subnetVpnId);
2965                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2966             }
2967         }
2968     }
2969 }