natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.JdkFutureAdapters;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import java.math.BigInteger;
  18 import java.net.Inet6Address;
  19 import java.net.InetAddress;
  20 import java.net.UnknownHostException;
  21 import java.util.ArrayList;
  22 import java.util.Arrays;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.HashMap;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Optional;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PreDestroy;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  40 import org.opendaylight.genius.infra.Datastore.Configuration;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  44 import org.opendaylight.genius.infra.TypedWriteTransaction;
  45 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  46 import org.opendaylight.genius.mdsalutil.ActionInfo;
  47 import org.opendaylight.genius.mdsalutil.BucketInfo;
  48 import org.opendaylight.genius.mdsalutil.FlowEntity;
  49 import org.opendaylight.genius.mdsalutil.GroupEntity;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  54 import org.opendaylight.genius.mdsalutil.NwConstants;
  55 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  68 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  69 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  73 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  75 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  76 import org.opendaylight.infrautils.utils.concurrent.Executors;
  77 import org.opendaylight.mdsal.binding.api.DataBroker;
  78 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  79 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  80 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  81 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  82 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  83 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  84 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 147 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 148 import org.opendaylight.yangtools.yang.common.RpcResult;
 149 import org.opendaylight.yangtools.yang.common.Uint32;
 150 import org.opendaylight.yangtools.yang.common.Uint64;
 151 import org.slf4j.Logger;
 152 import org.slf4j.LoggerFactory;
 153
 154
 155 @Singleton
 156 public class ExternalRoutersListener extends AbstractAsyncDataTreeChangeListener<Routers> {
 157     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 158
 159     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 160     private static final Uint64 COOKIE_VM_LFIB_TABLE = Uint64.valueOf("8000022", 16).intern();
 161
 162     private final DataBroker dataBroker;
 163     private final ManagedNewTransactionRunner txRunner;
 164     private final IMdsalApiManager mdsalManager;
 165     private final ItmRpcService itmManager;
 166     private final OdlInterfaceRpcService odlInterfaceRpcService;
 167     private final IdManagerService idManager;
 168     private final NaptManager naptManager;
 169     private final NAPTSwitchSelector naptSwitchSelector;
 170     private final IBgpManager bgpManager;
 171     private final VpnRpcService vpnService;
 172     private final FibRpcService fibService;
 173     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 174     private final NaptEventHandler naptEventHandler;
 175     private final NaptPacketInHandler naptPacketInHandler;
 176     private final IFibManager fibManager;
 177     private final IVpnManager vpnManager;
 178     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 179     private final NatMode natMode;
 180     private final IElanService elanManager;
 181     private final JobCoordinator coordinator;
 182     private final IInterfaceManager interfaceManager;
 183     private final NatOverVxlanUtil natOverVxlanUtil;
 184     private final int snatPuntTimeout;
 185
 186     @Inject
 187     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 188                                    final ItmRpcService itmManager,
 189                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 190                                    final IdManagerService idManager,
 191                                    final NaptManager naptManager,
 192                                    final NAPTSwitchSelector naptSwitchSelector,
 193                                    final IBgpManager bgpManager,
 194                                    final VpnRpcService vpnService,
 195                                    final FibRpcService fibService,
 196                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 197                                    final NaptEventHandler naptEventHandler,
 198                                    final NaptPacketInHandler naptPacketInHandler,
 199                                    final IFibManager fibManager,
 200                                    final IVpnManager vpnManager,
 201                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 202                                    final NatserviceConfig config,
 203                                    final IElanService elanManager,
 204                                    final JobCoordinator coordinator,
 205                                    final NatOverVxlanUtil natOverVxlanUtil,
 206                                    final IInterfaceManager interfaceManager) {
 207         super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ExtRouters.class)
 208                 .child(Routers.class),
 209                 Executors.newListeningSingleThreadExecutor("ExternalRoutersListener", LOG));
 210         this.dataBroker = dataBroker;
 211         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 212         this.mdsalManager = mdsalManager;
 213         this.itmManager = itmManager;
 214         this.odlInterfaceRpcService = odlInterfaceRpcService;
 215         this.idManager = idManager;
 216         this.naptManager = naptManager;
 217         this.naptSwitchSelector = naptSwitchSelector;
 218         this.bgpManager = bgpManager;
 219         this.vpnService = vpnService;
 220         this.fibService = fibService;
 221         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 222         this.naptEventHandler = naptEventHandler;
 223         this.naptPacketInHandler = naptPacketInHandler;
 224         this.fibManager = fibManager;
 225         this.vpnManager = vpnManager;
 226         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 227         this.elanManager = elanManager;
 228         this.coordinator = coordinator;
 229         this.interfaceManager = interfaceManager;
 230         this.natOverVxlanUtil = natOverVxlanUtil;
 231         if (config != null) {
 232             this.natMode = config.getNatMode();
 233             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 234         } else {
 235             this.natMode = NatMode.Controller;
 236             this.snatPuntTimeout = 0;
 237         }
 238         init();
 239     }
 240
 241     public void init() {
 242         LOG.info("{} init", getClass().getSimpleName());
 243         // This class handles ExternalRouters for Controller SNAT mode.
 244         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 245         if (natMode == NatMode.Controller) {
 246             NatUtil.createGroupIdPool(idManager);
 247         }
 248     }
 249
 250     @Override
 251     @PreDestroy
 252     public void close() {
 253         super.close();
 254         Executors.shutdownAndAwaitTermination(getExecutorService());
 255     }
 256
 257     @Override
 258     // TODO Clean up the exception handling
 259     @SuppressWarnings("checkstyle:IllegalCatch")
 260     public void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 261         if (natMode != NatMode.Controller) {
 262             return;
 263         }
 264         // Populate the router-id-name container
 265         String routerName = routers.getRouterName();
 266         LOG.info("add : external router event for {}", routerName);
 267         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
 268         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 269         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 270         try {
 271             if (routers.isEnableSnat()) {
 272                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 273                     () -> Collections.singletonList(
 274                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 275                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 276                             Uint32 bgpVpnId = NatConstants.INVALID_ID;
 277                             if (bgpVpnUuid != null) {
 278                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 279                             }
 280                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 281                             // Allocate Primary Napt Switch for this router
 282                             Uint64 primarySwitchId = getPrimaryNaptSwitch(routerName);
 283                             if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 284                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 285                             }
 286                         }
 287                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 288             } else {
 289                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 290             }
 291         } catch (Exception ex) {
 292             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 293                     routerName, ex);
 294         }
 295     }
 296
 297     public void handleEnableSnat(Routers routers, Uint32 routerId, Uint64 primarySwitchId, Uint32 bgpVpnId,
 298                                  TypedWriteTransaction<Configuration> confTx) {
 299         String routerName = routers.getRouterName();
 300         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 301
 302         naptManager.initialiseExternalCounter(routers, routerId);
 303         subnetRegisterMapping(routers, routerId);
 304
 305         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 306             primarySwitchId, routerName);
 307
 308         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 309                 routers.getNetworkId());
 310         if (extNwProvType == null) {
 311             LOG.error("handleEnableSnat : External Network Provider Type missing");
 312             return;
 313         }
 314
 315         if (bgpVpnId != NatConstants.INVALID_ID) {
 316             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId,
 317                 routers.getNetworkId(),false, confTx, extNwProvType);
 318         } else {
 319             // write metadata and punt
 320             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 321             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, routers.getNetworkId(), confTx);
 322             // Now install entries in SNAT tables to point to Primary for each router
 323             List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 324             for (Uint64 dpnId : switches) {
 325                 // Handle switches and NAPT switches separately
 326                 if (!dpnId.equals(primarySwitchId)) {
 327                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 328                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 329                 }
 330             }
 331         }
 332
 333         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 334         if (externalIps.isEmpty()) {
 335             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 336             return;
 337         } else {
 338             for (String externalIpAddrPrefix : externalIps) {
 339                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 340                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 341                 externalIpAddrPrefix = NatUtil.validateAndAddNetworkMask(externalIpAddrPrefix);
 342                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 343                 );
 344             }
 345         }
 346         LOG.debug("handleEnableSnat : Exit");
 347     }
 348
 349     private Uint64 getPrimaryNaptSwitch(String routerName) {
 350         // Allocate Primary Napt Switch for this router
 351         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 352         if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 353             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 354                 primarySwitchId, routerName);
 355             return primarySwitchId;
 356         }
 357         return selectNewNAPTSwitch(routerName);
 358     }
 359
 360     private Uint64 selectNewNAPTSwitch(String routerName) {
 361         // Allocated an id from VNI pool for the Router.
 362         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 363         Uint64 primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName, null);
 364         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 365
 366         return primarySwitchId;
 367     }
 368
 369     protected void installNaptPfibExternalOutputFlow(String routerName, Uint32 routerId, Uint64 dpnId,
 370                                                      TypedWriteTransaction<Configuration> confTx) {
 371         Uint32 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 372         if (extVpnId == NatConstants.INVALID_ID) {
 373             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 374             extVpnId = routerId;
 375         }
 376         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 377         if (externalIps.isEmpty()) {
 378             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 379                 dpnId, extVpnId);
 380             return;
 381         }
 382         for (String ip : externalIps) {
 383             Uuid subnetId = getSubnetIdForFixedIp(ip);
 384             if (subnetId != null) {
 385                 Uint32 subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 386                 if (subnetVpnId != NatConstants.INVALID_ID) {
 387                     extVpnId = subnetVpnId;
 388                 }
 389                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 390                     dpnId, extVpnId, subnetId);
 391                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 392                 if (postNaptFlowEntity != null) {
 393                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 394                 }
 395             }
 396         }
 397     }
 398
 399     @Nullable
 400     private Uuid getSubnetIdForFixedIp(String ip) {
 401         if (ip != null) {
 402             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 403             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 404             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 405         }
 406         LOG.error("getSubnetIdForFixedIp : ip is null");
 407         return null;
 408     }
 409
 410     protected void subnetRegisterMapping(Routers routerEntry, Uint32 segmentId) {
 411         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 412         List<String> externalIps = NatUtil.getIpsListFromExternalIps(
 413                 new ArrayList<ExternalIps>(routerEntry.getExternalIps().values()));
 414         int counter = 0;
 415         int extIpCounter = externalIps.size();
 416         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 417                 counter, extIpCounter);
 418         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 419         if (subnetIds == null) {
 420             return;
 421         }
 422         for (Uuid subnet : subnetIds) {
 423             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 424             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 425                 .builder(Subnetmaps.class)
 426                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 427                 .build();
 428             Optional<Subnetmap> sn;
 429             try {
 430                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 431                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 432             } catch (InterruptedException | ExecutionException e) {
 433                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 434                 sn = Optional.empty();
 435             }
 436             if (sn.isPresent()) {
 437                 // subnets
 438                 Subnetmap subnetmapEntry = sn.get();
 439                 String subnetString = subnetmapEntry.getSubnetIp();
 440                 String[] subnetSplit = subnetString.split("/");
 441                 String subnetIp = subnetSplit[0];
 442                 try {
 443                     InetAddress address = InetAddress.getByName(subnetIp);
 444                     if (address instanceof Inet6Address) {
 445                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 446                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 447                         continue;
 448                     }
 449                 } catch (UnknownHostException e) {
 450                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 451                     return;
 452                 }
 453                 String subnetPrefix = "0";
 454                 if (subnetSplit.length == 2) {
 455                     subnetPrefix = subnetSplit[1];
 456                 }
 457                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 458                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 459                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 460                 //externalIps
 461                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 462                         counter, extIpCounter);
 463                 if (extIpCounter != 0) {
 464                     if (counter < extIpCounter) {
 465                         String[] ipSplit = externalIps.get(counter).split("/");
 466                         String externalIp = ipSplit[0];
 467                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 468                         if (ipSplit.length == 2) {
 469                             extPrefix = ipSplit[1];
 470                         }
 471                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 472                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 473                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 474                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 475                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 476                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 477                     } else {
 478                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 479                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 480                         String[] ipSplit = externalIps.get(counter).split("/");
 481                         String externalIp = ipSplit[0];
 482                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 483                         if (ipSplit.length == 2) {
 484                             extPrefix = ipSplit[1];
 485                         }
 486                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 487                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 488                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 489                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 490                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 491                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 492                             externalIp, extPrefix);
 493                     }
 494                 }
 495                 counter++;
 496                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 497             } else {
 498                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 499             }
 500         }
 501     }
 502
 503     private void addOrDelDefFibRouteToSNAT(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 504             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 505             throws ExecutionException, InterruptedException {
 506         //Check if BGP VPN exists. If exists then invoke the new method.
 507         if (bgpVpnId != NatConstants.INVALID_ID) {
 508             if (bgpVpnUuid != null) {
 509                 String bgpVpnName = bgpVpnUuid.getValue();
 510                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 511                     bgpVpnId, bgpVpnName);
 512                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 513                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 514                 confTx.mergeParentStructurePut(getRoutersIdentifier(bgpVpnId), rtrs);
 515             }
 516             if (create) {
 517                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 518             } else {
 519                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 520             }
 521             return;
 522         }
 523
 524         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 525         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 526     }
 527
 528     private void addOrDelDefaultFibRouteForSNAT(String routerName, Uint32 routerId, boolean create,
 529             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 530         List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 531         if (switches.isEmpty()) {
 532             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 533             return;
 534         }
 535         if (routerId == NatConstants.INVALID_ID) {
 536             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 537                     + "default NAT route in FIB", routerName);
 538             return;
 539         }
 540         for (Uint64 dpnId : switches) {
 541             if (create) {
 542                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 543                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 544                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 545             } else {
 546                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 547                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 548                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 549             }
 550         }
 551     }
 552
 553     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId,
 554                                                      Uint32 bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 555         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 556         if (dpnIds.isEmpty()) {
 557             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 558                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 559             return;
 560         }
 561         for (Uint64 dpnId : dpnIds) {
 562             if (bgpVpnId != NatConstants.INVALID_ID) {
 563                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 564                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 565                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 566             } else {
 567                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 568                     + "in dpn {} for the internal vpn", routerId, dpnId);
 569                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 570             }
 571         }
 572     }
 573
 574     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 575                                                         TypedReadWriteTransaction<Configuration> confTx)
 576             throws ExecutionException, InterruptedException {
 577         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 578         if (dpnIds.isEmpty()) {
 579             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 580                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 581             return;
 582         }
 583         for (Uint64 dpnId : dpnIds) {
 584             if (bgpVpnId != NatConstants.INVALID_ID) {
 585                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 586                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 587                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 588             } else {
 589                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 590                     + "in dpn {} for the internal vpn", routerId, dpnId);
 591                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 592             }
 593         }
 594     }
 595
 596     protected void installOutboundMissEntry(String routerName, Uint32 routerId, Uint64 primarySwitchId,
 597         TypedWriteTransaction<Configuration> confTx) {
 598         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 599         if (routerId != NatConstants.INVALID_ID) {
 600             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 601                     primarySwitchId, routerId);
 602             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 603         } else {
 604             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 605                 + "createAndInstallMissEntry", routerName);
 606         }
 607     }
 608
 609     public String getFlowRefOutbound(Uint64 dpnId, short tableId, Uint32 routerID, int protocol) {
 610         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 611                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 612     }
 613
 614     private String getFlowRefNaptPreFib(Uint64 dpnId, short tableId, Uint32 vpnId) {
 615         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 616                 .FLOWID_SEPARATOR + vpnId;
 617     }
 618
 619     public Uint64 getCookieOutboundFlow(Uint32 routerId) {
 620         return Uint64.valueOf((NwConstants.COOKIE_OUTBOUND_NAPT_TABLE).toJava().add(
 621                 new BigInteger("0110001", 16)).add(BigInteger.valueOf(routerId.longValue())));
 622     }
 623
 624     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, Uint64 cookie) {
 625         long l4SrcPortField;
 626         long l4DstPortField;
 627         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 628
 629         if (protocol == NwConstants.IP_PROT_TCP) {
 630             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 631             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 632         } else {
 633             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 634             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 635         }
 636         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 637                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 638                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 639                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 640                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 641                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 642                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 643                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 644                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 645                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 646                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 647                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 648                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 649                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 650                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 651
 652         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 653                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 654     }
 655
 656     private FlowEntity buildIcmpDropFlow(Uint64 dpnId, Uint32 routerId, Uint32 vpnId) {
 657         List<MatchInfo> matches = new ArrayList<>();
 658         matches.add(MatchEthernetType.IPV4);
 659         matches.add(MatchIpProtocol.ICMP);
 660         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 661                 MetaDataUtil.METADATA_MASK_VRFID));
 662
 663         List<ActionInfo> actionInfos = new ArrayList<>();
 664         actionInfos.add(new ActionDrop());
 665
 666         List<InstructionInfo> instructions = new ArrayList<>();
 667         instructions.add(new InstructionApplyActions(actionInfos));
 668
 669         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 670                 NwConstants.IP_PROT_ICMP);
 671         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 672                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 673                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 674         return flow;
 675     }
 676
 677     protected FlowEntity buildOutboundFlowEntity(Uint64 dpId, Uint32 routerId, int protocol) {
 678         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 679         Uint64 cookie = getCookieOutboundFlow(routerId);
 680         List<MatchInfo> matches = new ArrayList<>();
 681         matches.add(MatchEthernetType.IPV4);
 682         matches.add(new MatchIpProtocol((short)protocol));
 683         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 684                 MetaDataUtil.METADATA_MASK_VRFID));
 685
 686         List<InstructionInfo> instructions = new ArrayList<>();
 687         List<ActionInfo> actionsInfos = new ArrayList<>();
 688         actionsInfos.add(new ActionPuntToController());
 689         if (snatPuntTimeout != 0) {
 690             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 691         }
 692         instructions.add(new InstructionApplyActions(actionsInfos));
 693
 694         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 695         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 696             5, flowRef, 0, 0,
 697             cookie, matches, instructions);
 698         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 699         return flowEntity;
 700     }
 701
 702     public void createOutboundTblEntry(Uint64 dpnId, Uint32 routerId, TypedWriteTransaction<Configuration> confTx) {
 703         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 704         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 705         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 706         mdsalManager.addFlow(confTx, tcpflowEntity);
 707
 708         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 709         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 710         mdsalManager.addFlow(confTx, udpflowEntity);
 711
 712         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 713         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 714         mdsalManager.addFlow(confTx, icmpDropFlow);
 715     }
 716
 717     @Nullable
 718     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 719         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 720         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 721         try {
 722             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 723                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 724                     .setSourceDpid(srcDpId)
 725                     .setDestinationDpid(dstDpId)
 726                     .setTunnelType(tunType)
 727                     .build());
 728             rpcResult = result.get();
 729             if (!rpcResult.isSuccessful()) {
 730                 tunType = TunnelTypeGre.class;
 731                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 732                     .setSourceDpid(srcDpId)
 733                     .setDestinationDpid(dstDpId)
 734                     .setTunnelType(tunType)
 735                     .build());
 736                 rpcResult = result.get();
 737                 if (!rpcResult.isSuccessful()) {
 738                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 739                             rpcResult.getErrors());
 740                 } else {
 741                     return rpcResult.getResult().getInterfaceName();
 742                 }
 743                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 744                         rpcResult.getErrors());
 745             } else {
 746                 return rpcResult.getResult().getInterfaceName();
 747             }
 748         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 749             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 750                     + "between {} and {}", srcDpId, dstDpId, e);
 751         }
 752
 753         return null;
 754     }
 755
 756     protected void installSnatMissEntryForPrimrySwch(Uint64 dpnId, String routerName, Uint32 routerId,
 757         TypedWriteTransaction<Configuration> confTx) {
 758
 759         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 760         // Install miss entry pointing to group
 761         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 762         mdsalManager.addFlow(confTx, flowEntity);
 763     }
 764
 765     protected void installSnatMissEntry(Uint64 dpnId, List<BucketInfo> bucketInfo,
 766             String routerName, Uint32 routerId) {
 767         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 768             dpnId, bucketInfo.get(0));
 769         // Install the select group
 770         Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 771             NatUtil.getGroupIdKey(routerName));
 772         if (groupId == NatConstants.INVALID_ID) {
 773             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 774             return;
 775         }
 776         GroupEntity groupEntity =
 777             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 778         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 779         mdsalManager.syncInstallGroup(groupEntity);
 780         // Install miss entry pointing to group
 781         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 782         if (flowEntity == null) {
 783             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 784                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 785                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 786             return;
 787         }
 788         mdsalManager.installFlow(flowEntity);
 789     }
 790
 791     void installGroup(Uint64 dpnId, String routerName, Uint32 groupId, List<BucketInfo> bucketInfo) {
 792         GroupEntity groupEntity =
 793             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 794         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 795         mdsalManager.syncInstallGroup(groupEntity);
 796     }
 797
 798     private FlowEntity buildSnatFlowEntity(Uint64 dpId, String routerName, Uint32 routerId, Uint32 groupId) {
 799         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 800                 dpId, routerName, groupId);
 801         List<MatchInfo> matches = new ArrayList<>();
 802         matches.add(MatchEthernetType.IPV4);
 803         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 804                 MetaDataUtil.METADATA_MASK_VRFID));
 805
 806         List<ActionInfo> actionsInfo = new ArrayList<>();
 807         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 808                 routerId, routerName);
 809         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
 810         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 811         actionsInfo.add(new ActionGroup(groupId.longValue()));
 812         List<InstructionInfo> instructions = new ArrayList<>();
 813         instructions.add(new InstructionApplyActions(actionsInfo));
 814         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 815         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 816                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 817                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 818
 819         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 820         return flowEntity;
 821     }
 822
 823     private FlowEntity buildSnatFlowEntityForPrmrySwtch(Uint64 dpId, String routerName, Uint32 routerId) {
 824
 825         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 826             routerName);
 827         List<MatchInfo> matches = new ArrayList<>();
 828         matches.add(MatchEthernetType.IPV4);
 829         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 830                 MetaDataUtil.METADATA_MASK_VRFID));
 831
 832         List<InstructionInfo> instructions = new ArrayList<>();
 833         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 834
 835         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 836         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 837             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 838             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 839
 840         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 841         return flowEntity;
 842     }
 843
 844     // TODO : Replace this with ITM Rpc once its available with full functionality
 845     protected void installTerminatingServiceTblEntry(Uint64 dpnId, String routerName, Uint32 routerId,
 846         TypedWriteTransaction<Configuration> confTx) {
 847
 848         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 849             dpnId, routerName);
 850         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 851         if (flowEntity == null) {
 852             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 853                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 854                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 855                     dpnId, routerName);
 856             return;
 857         }
 858         mdsalManager.addFlow(confTx, flowEntity);
 859
 860     }
 861
 862     private FlowEntity buildTsFlowEntity(Uint64 dpId, String routerName, Uint32 routerId) {
 863         List<MatchInfo> matches = new ArrayList<>();
 864         matches.add(MatchEthernetType.IPV4);
 865         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 866                 idManager, routerId, routerName);
 867         matches.add(new MatchTunnelId(tunnelId));
 868         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
 869         List<InstructionInfo> instructions = new ArrayList<>();
 870         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 871                 MetaDataUtil.METADATA_MASK_VRFID));
 872         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 873         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 874                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
 875                 NwConstants.COOKIE_TS_TABLE, matches, instructions);
 876         return flowEntity;
 877     }
 878
 879     public String getFlowRefTs(Uint64 dpnId, short tableId, Uint32 routerID) {
 880         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 881                 .FLOWID_SEPARATOR + routerID;
 882     }
 883
 884     public static String getFlowRefSnat(Uint64 dpnId, short tableId, String routerID) {
 885         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 886                 .FLOWID_SEPARATOR + routerID;
 887     }
 888
 889     protected void handleSwitches(Uint64 dpnId, String routerName, Uint32 routerId, Uint64 primarySwitchId) {
 890         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 891         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 892         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 893         List<BucketInfo> listBucketInfo = new ArrayList<>();
 894
 895         if (ifNamePrimary != null) {
 896             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 897             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 898                     interfaceManager, ifNamePrimary, routerId, true);
 899             if (listActionInfoPrimary.isEmpty()) {
 900                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 901                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 902                         ifNamePrimary);
 903             }
 904         } else {
 905             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 906                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 907         }
 908         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 909
 910         listBucketInfo.add(0, bucketPrimary);
 911         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 912     }
 913
 914     List<BucketInfo> getBucketInfoForNonNaptSwitches(Uint64 nonNaptSwitchId,
 915                                                      Uint64 primarySwitchId, String routerName, Uint32 routerId) {
 916         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 917         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 918         List<BucketInfo> listBucketInfo = new ArrayList<>();
 919
 920         if (ifNamePrimary != null) {
 921             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 922                     ifNamePrimary);
 923             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 924                     interfaceManager, ifNamePrimary, routerId, true);
 925             if (listActionInfoPrimary.isEmpty()) {
 926                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 927                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 928                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 929             }
 930         } else {
 931             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 932                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 933         }
 934         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 935
 936         listBucketInfo.add(0, bucketPrimary);
 937         return listBucketInfo;
 938     }
 939
 940     protected void handlePrimaryNaptSwitch(Uint64 dpnId, String routerName, Uint32 routerId, Uuid externalNwUuid,
 941         TypedWriteTransaction<Configuration> confTx) {
 942
 943        /*
 944         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 945         */
 946
 947         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 948
 949 /*
 950         List<BucketInfo> listBucketInfo = new ArrayList<>();
 951         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 952         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 953         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 954         listBucketInfo.add(0, bucketPrimary);
 955 */
 956
 957         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 958         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 959         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 960         installNaptPfibEntry(dpnId, routerId, confTx);
 961         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 962         if (networkId != null) {
 963             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 964             if (vpnUuid != null) {
 965                 Uint32 extVpnId = NatUtil.getExternalVpnIdForExtNetwork(dataBroker, externalNwUuid);
 966                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 967                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 968                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 969                     if (extVpnId != null && extVpnId != NatConstants.INVALID_ID) {
 970                         installNaptPfibEntry(dpnId, extVpnId, null);
 971                     }
 972                     return Collections.emptyList();
 973                 });
 974             } else {
 975                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 976             }
 977         } else {
 978             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 979         }
 980     }
 981
 982     public void installNaptPfibEntry(Uint64 dpnId, Uint32 segmentId,
 983             @Nullable TypedWriteTransaction<Configuration> confTx) {
 984         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 985         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 986         if (confTx != null) {
 987             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 988         } else {
 989             mdsalManager.installFlow(naptPfibFlowEntity);
 990         }
 991     }
 992
 993     public FlowEntity buildNaptPfibFlowEntity(Uint64 dpId, Uint32 segmentId) {
 994
 995         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 996         List<MatchInfo> matches = new ArrayList<>();
 997         matches.add(MatchEthernetType.IPV4);
 998         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId.longValue()),
 999                 MetaDataUtil.METADATA_MASK_VRFID));
1000
1001         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1002         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1003         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
1004         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1005         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1006
1007         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1008         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1009             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1010             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1011         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1012         return flowEntity;
1013     }
1014
1015     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, Uint64 dpnId, Routers router,
1016                                          Uint32 routerId, String routerName, String externalIp) {
1017         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1018             dpnId, routerId, externalIp);
1019         Uuid networkId = router.getNetworkId();
1020         if (networkId == null) {
1021             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1022             return;
1023         }
1024         Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(
1025                 new ArrayList<ExternalIps>(router.getExternalIps().values()));
1026         // FLAT/VLAN case having external-subnet as VPN
1027         String externalSubnetVpn = null;
1028         if (externalSubnetList != null && !externalSubnetList.isEmpty()) {
1029             Boolean isExternalIpsAdvertized = Boolean.FALSE;
1030             for (Uuid externalSubnetId : externalSubnetList) {
1031                 Optional<Subnets> externalSubnet = NatUtil
1032                     .getOptionalExternalSubnets(dataBroker, externalSubnetId);
1033                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1034                 if (externalSubnet.isPresent()) {
1035                     externalSubnetVpn = externalSubnetId.getValue();
1036                     advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1037                         externalSubnetVpn, routerId, routerName,
1038                         externalIp, networkId, router, confTx);
1039                     isExternalIpsAdvertized = Boolean.TRUE;
1040                 }
1041             }
1042             if (isExternalIpsAdvertized) {
1043                 LOG.trace("External Ips {} advertized for Router {}", router.getExternalIps(), routerName);
1044                 return;
1045             }
1046         }
1047
1048         // VXVLAN/GRE case having Internet-VPN
1049         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1050         if (vpnName == null) {
1051             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1052                     + "configuration {} in router {}", networkId, externalIp, routerId);
1053             return;
1054         }
1055         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1056             externalIp, networkId, router, confTx);
1057         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1058             dpnId, routerId, externalIp);
1059     }
1060
1061     public void advToBgpAndInstallFibAndTsFlows(final Uint64 dpnId, final short tableId, final String vpnName,
1062                                                 final Uint32 routerId, final String routerName, final String externalIp,
1063                                                 final Uuid extNetworkId, @Nullable final Routers router,
1064                                                 final TypedWriteTransaction<Configuration> confTx) {
1065         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1066                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1067         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1068         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1069         if (rd == null || rd.isEmpty()) {
1070             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1071             return;
1072         }
1073         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1074         if (extNwProvType == null) {
1075             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1076             return;
1077         }
1078         if (extNwProvType == ProviderTypes.VXLAN) {
1079             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1080                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1081             return;
1082         }
1083         //Generate VPN label for the external IP
1084         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1085             .setIpPrefix(externalIp).build();
1086         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1087
1088         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1089         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1090             if (result.isSuccessful()) {
1091                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1092                 GenerateVpnLabelOutput output = result.getResult();
1093                 final Uint32 label = output.getLabel();
1094
1095                 int externalIpInDsFlag = 0;
1096                 //Get IPMaps from the DB for the router ID
1097                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1098                 for (IpMap dbIpMap : dbIpMaps) {
1099                     String dbExternalIp = dbIpMap.getExternalIp();
1100                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1101                     if (dbExternalIp.contains(externalIp)) {
1102                         String dbInternalIp = dbIpMap.getInternalIp();
1103                         IpMapKey dbIpMapKey = dbIpMap.key();
1104                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1105                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1106                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1107                             .setExternalIp(dbExternalIp).setLabel(label).build();
1108                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1109                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1110                         externalIpInDsFlag++;
1111                     }
1112                 }
1113                 if (externalIpInDsFlag <= 0) {
1114                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1115                             + "Failed to update label {} for routerId {} in DS",
1116                             externalIp, label, routerId);
1117                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1118                         + " found in DS for router %s", label, externalIp, routerId);
1119                     return Futures.immediateFailedFuture(new Exception(errMsg));
1120                 }
1121                 //Inform BGP
1122                 Uint32 l3vni = Uint32.ZERO;
1123                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1124                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
1125                 }
1126                 Routers extRouter = router != null ? router :
1127                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1128                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1129                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1130                     RouteOrigin.STATIC, dpnId);
1131
1132                 //Install custom FIB routes
1133                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1134                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1135                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1136                         extNwProvType);
1137                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1138
1139                 //Install custom FIB routes - FIB table.
1140                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1141                         routerName, externalIp);
1142                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1143                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1144                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1145                 }
1146                 String externalVpn = vpnName;
1147                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1148                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1149                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1150                             externalSubnetId);
1151                     if (externalSubnet.isPresent()) {
1152                         externalVpn =  externalSubnetId.getValue();
1153                     }
1154                 }
1155                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1156                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1157                     .setVpnName(externalVpn)
1158                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1159                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1160                     .setInstruction(fibTableCustomInstructions).build();
1161                 return fibService.createFibEntry(input);
1162             } else {
1163                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1164                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1165                     externalIp, vpnName, result.getErrors());
1166                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1167             }
1168         }, MoreExecutors.directExecutor());
1169
1170         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1171
1172             @Override
1173             public void onFailure(@NonNull Throwable error) {
1174                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1175             }
1176
1177             @Override
1178             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1179                 if (result.isSuccessful()) {
1180                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1181                             externalIp);
1182                 } else {
1183                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1184                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1185                 }
1186             }
1187         }, MoreExecutors.directExecutor());
1188     }
1189
1190     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1191             String externalIp) {
1192         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1193         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1194         Uint32 externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1195                 externalIp, router);
1196         int instructionIndex = 0;
1197         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1198             Uint64 subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId.longValue());
1199             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1200                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1201             instructionIndex++;
1202         }
1203
1204         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1205         return fibTableCustomInstructions;
1206     }
1207
1208     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, short tableId,
1209         TypedWriteTransaction<Configuration> confTx) {
1210         List<MatchInfo> matches = new ArrayList<>();
1211         matches.add(MatchEthernetType.MPLS_UNICAST);
1212         matches.add(new MatchMplsLabel(serviceId.longValue()));
1213
1214         Map<InstructionKey, Instruction> instructionsMap = new HashMap<InstructionKey, Instruction>();
1215         int instructionKey = 0;
1216         List<ActionInfo> actionsInfos = new ArrayList<>();
1217         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1218         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1219         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1220         instructionsMap.put(new InstructionKey(++instructionKey), writeInstruction);
1221         instructionsMap.put(new InstructionKey(++instructionKey),
1222                 new InstructionGotoTable(tableId).buildInstruction(1));
1223
1224         // Install the flow entry in L3_LFIB_TABLE
1225         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1226
1227         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1228             10, flowRef, 0, 0,
1229             COOKIE_VM_LFIB_TABLE, matches, instructionsMap);
1230
1231         mdsalManager.addFlow(confTx, dpId, flowEntity);
1232
1233         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1234     }
1235
1236     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId, Uint32 l3Vni,
1237         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1238         ProviderTypes extNwProvType) {
1239
1240         List<MatchInfo> mkMatches = new ArrayList<>();
1241
1242         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1243             dpnId, serviceId, customInstructions);
1244
1245         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1246             mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
1247         } else {
1248             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
1249         }
1250         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<InstructionKey, Instruction>();
1251         int instructionKey = 0;
1252         for (Instruction instructionObj : customInstructions) {
1253             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
1254         }
1255         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1256             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1257                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1258                 String.format("%s:%s", "TST Flow Entry ", serviceId), 0, 0,
1259                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(serviceId.longValue()))),
1260                 mkMatches, customInstructionsMap);
1261
1262         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1263     }
1264
1265     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(Uint32 routerId) {
1266         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1267             new RouterIdsKey(routerId)).build();
1268     }
1269
1270     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
1271         return NatConstants.SNAT_FLOWID_PREFIX + dpnId.toString() + NwConstants.FLOWID_SEPARATOR + tableId
1272                 + NwConstants.FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1273     }
1274
1275     @Override
1276     public void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1277         if (natMode != NatMode.Controller) {
1278             return;
1279         }
1280         LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1281         String routerName = original.getRouterName();
1282         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1283         if (routerId == NatConstants.INVALID_ID) {
1284             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1285             return;
1286         }
1287         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1288             List<ListenableFuture<Void>> futures = new ArrayList<>();
1289             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1290                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1291                     Uint32 bgpVpnId = NatConstants.INVALID_ID;
1292                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1293                     if (bgpVpnUuid != null) {
1294                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1295                     }
1296                     //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1297                     /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1298                      * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1299                      */
1300                     Uint64 dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1301                     boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(Uint64
1302                             .valueOf(BigInteger.ZERO)));
1303                     Uuid networkId = original.getNetworkId();
1304                     // Check if its update on SNAT flag
1305                     boolean originalSNATEnabled = original.isEnableSnat();
1306                     boolean updatedSNATEnabled = update.isEnableSnat();
1307                     LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1308                             + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1309                         originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1310                     // Cluster Reboot Case Handling
1311                     // 1. DPN not elected during add event(due to none of the OVS connected)
1312                     // 2. Update event called with changes of parameters(but enableSnat is not changed)
1313                     // 3. First Elect dpnId and process other changes with valid dpnId
1314                     if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1315                         if (originalSNATEnabled && !updatedSNATEnabled) {
1316                             if (isPrimaryNaptSwitchNotSelected) {
1317                                 LOG.info("No Action to be taken when SNAT is disabled "
1318                                     + "with no Napt Switch Election for Router {}", routerName);
1319                                 return;
1320                             }
1321                             //SNAT disabled for the router
1322                             Uuid networkUuid = original.getNetworkId();
1323                             LOG.info("update : SNAT disabled for Router {}", routerName);
1324                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1325                             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1326                             handleDisableSnat(original, networkUuid, externalIps, false, vpnName,
1327                                     dpnId, routerId, removeFlowInvTx);
1328                         }  else if (updatedSNATEnabled) {
1329                             LOG.info("update : SNAT enabled for Router {}", routerName);
1330                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1331                                 true, writeFlowInvTx);
1332                             if (isPrimaryNaptSwitchNotSelected) {
1333                                 dpnId = selectNewNAPTSwitch(routerName);
1334                                 if (dpnId != null && !dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1335                                     handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1336                                 } else {
1337                                     LOG.error("update : Failed to elect Napt Switch During update event"
1338                                         + " of router {}", routerName);
1339                                 }
1340                             }
1341                         }
1342                         LOG.info("update : no need to process external/subnet changes as it's will taken care"
1343                             + "in handleDisableSnat/handleEnableSnat");
1344                         return;
1345                     }
1346                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1347                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1348                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1349                     }
1350
1351                     if (updatedSNATEnabled != originalSNATEnabled) {
1352                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1353                                 + "handleDisableSnat/handleEnableSnat");
1354                         return;
1355                     }
1356                     //Check if the Update is on External IPs
1357                     LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1358                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(
1359                             new ArrayList<ExternalIps>(original.getExternalIps().values()));
1360                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(
1361                             new ArrayList<ExternalIps>(update.getExternalIps().values()));
1362
1363                     //Check if the External IPs are removed during the update.
1364                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1365                     removedExternalIps.removeAll(updatedExternalIps);
1366                     if (removedExternalIps.size() > 0) {
1367                         LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1368                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1369                                 removedExternalIps, original.getExtGwMacAddress(),
1370                                 dpnId, networkId);
1371
1372                         for (String removedExternalIp : removedExternalIps) {
1373                 /*
1374                     1) Remove the mappings in the IntExt IP model which has external IP.
1375                     2) Remove the external IP in the ExternalCounter model.
1376                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1377                        least loaded external IP.
1378                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1379                     4) Increase the count of the allocated external IP by one.
1380                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1381                      i.e. the route for the external IP is absent.
1382                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1383                      the removed external IPs and also from the model.
1384                     7) Advertise to the BGP for removing the route for the removed external IPs.
1385                  */
1386
1387                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1388                             String externalIp = externalIpParts[0];
1389                             String externalIpPrefix = externalIpParts[1];
1390                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1391
1392                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1393                                     + "entries for removed external IP {}", externalIpAddrStr);
1394                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1395                             String vpnName = "";
1396                             if (vpnUuId != null) {
1397                                 vpnName = vpnUuId.getValue();
1398                             }
1399                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1400                                     update.getExtGwMacAddress(), removeFlowInvTx);
1401
1402                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1403                             //Get the internal IPs which are associated to the removed external IPs
1404                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1405                             List<String> removedInternalIps = new ArrayList<>();
1406                             for (IpMap ipMap : ipMaps) {
1407                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1408                                     removedInternalIps.add(ipMap.getInternalIp());
1409                                 }
1410                             }
1411
1412                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1413                             for (String removedInternalIp : removedInternalIps) {
1414                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1415                                                 + "router ID {} from the IntExtIP model",
1416                                         removedInternalIp, routerId);
1417                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1418                             }
1419
1420                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1421                                             + "router ID {} from the ExternalIpsCounter model.",
1422                                     externalIpAddrStr, routerId);
1423                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1424
1425                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1426                                     + "whose external IPs were removed.");
1427                             for (String removedInternalIp : removedInternalIps) {
1428                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1429                                         removedInternalIp, writeFlowInvTx);
1430                             }
1431                             LOG.debug("update : Remove the NAPT translation entries from "
1432                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1433                             //Get the internalIP and internal Port which were associated to the removed external IP.
1434                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1435                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1436                                     .builder(IntextIpPortMap.class)
1437                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1438                             Optional<IpPortMapping> ipPortMapping;
1439                             try {
1440                                 ipPortMapping = SingleTransactionDataBroker
1441                                             .syncReadOptional(dataBroker,
1442                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1443                             } catch (InterruptedException | ExecutionException e) {
1444                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1445                                 ipPortMapping = Optional.empty();
1446                             }
1447
1448                             if (ipPortMapping.isPresent()) {
1449                                 for (IntextIpProtocolType intextIpProtocolType :
1450                                         ipPortMapping.get().nonnullIntextIpProtocolType().values()) {
1451                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1452                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap().values()) {
1453                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1454                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1455                                             List<String> removedInternalIpPorts =
1456                                                     protoTypesIntIpPortsMap.get(protoType);
1457                                             if (removedInternalIpPorts != null) {
1458                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1459                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1460                                             } else {
1461                                                 removedInternalIpPorts = new ArrayList<>();
1462                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1463                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1464                                             }
1465                                         }
1466                                     }
1467                                 }
1468                             }
1469
1470                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1471                             // the removed external IP.
1472                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1473                                     protoTypesIntIpPortsMap.entrySet();
1474                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1475                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1476                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1477                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1478                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1479                                     // Remove the IP port map from the intext-ip-port-map model,
1480                                     // which were containing the removed external IP
1481                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1482                                             protocolType);
1483                                     //Remove the IP port incomint packer map.
1484                                     naptPacketInHandler.removeIncomingPacketMap(
1485                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1486                                     String[] removedInternalIpPortParts = removedInternalIpPort
1487                                             .split(NatConstants.COLON_SEPARATOR);
1488                                     if (removedInternalIpPortParts.length == 2) {
1489                                         String removedInternalIp = removedInternalIpPortParts[0];
1490                                         String removedInternalPort = removedInternalIpPortParts[1];
1491                                         List<String> removedInternalPortsList =
1492                                                 internalIpPortMap.get(removedInternalPort);
1493                                         if (removedInternalPortsList != null) {
1494                                             removedInternalPortsList.add(removedInternalPort);
1495                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1496                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1497                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1498                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1499                                             //Remove the NAPT translation entries from Outbound NAPT table
1500                                             naptEventHandler.removeNatFlows(dpnId,
1501                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1502                                                 routerId, removedInternalIp,
1503                                                 Integer.parseInt(removedInternalPort),
1504                                                 protocolType.getName());
1505                                             naptEventHandler.removeNatFlows(dpnId,
1506                                                 NwConstants.INBOUND_NAPT_TABLE,
1507                                                 routerId, removedInternalIp,
1508                                                 Integer.parseInt(removedInternalPort),
1509                                                 protocolType.getName());
1510                                         } else {
1511                                             removedInternalPortsList = new ArrayList<>();
1512                                             removedInternalPortsList.add(removedInternalPort);
1513                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1514                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1515                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1516                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1517                                             //Remove the NAPT translation entries from Outbound NAPT table
1518                                             naptEventHandler.removeNatFlows(dpnId,
1519                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1520                                                 routerId, removedInternalIp,
1521                                                 Integer.parseInt(removedInternalPort),
1522                                                 protocolType.getName());
1523                                             naptEventHandler.removeNatFlows(dpnId,
1524                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1525                                                 Integer.parseInt(removedInternalPort),
1526                                                 protocolType.getName());
1527                                         }
1528                                     }
1529                                 }
1530                             }
1531
1532                             // Delete the entry from SnatIntIpPortMap DS
1533                             Set<String> internalIps = internalIpPortMap.keySet();
1534                             for (String internalIp : internalIps) {
1535                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1536                                         + "model SnatIntIpPortMap", internalIp);
1537                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1538                             }
1539
1540                             naptManager.removeNaptPortPool(externalIp);
1541                         }
1542                         LOG.debug(
1543                                 "update : End processing of the External IPs removal for router {}", routerName);
1544                     }
1545
1546                     //Check if the External IPs are added during the update.
1547                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1548                     addedExternalIps.removeAll(originalExternalIps);
1549                     if (addedExternalIps.size() != 0) {
1550                         LOG.debug("update : Start processing of the External IPs addition for router {}",
1551                             routerName);
1552                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1553                                 update.getExtGwMacAddress(), dpnId,
1554                                 update.getNetworkId());
1555
1556                         for (String addedExternalIp : addedExternalIps) {
1557                 /*
1558                     1) Do nothing in the IntExtIp model.
1559                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1560                  */
1561                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1562                             String externalIp = externalIpParts[0];
1563                             String externalIpPrefix = externalIpParts[1];
1564                             String externalpStr = externalIp + "/" + externalIpPrefix;
1565                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1566                                             + "router ID {} in the ExternalIpsCounter model.",
1567                                     externalpStr, routerId);
1568                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1569                             subnetRegisterMapping(update, routerId);
1570                             LOG.info("update : Installing fib flow fo newly added Ips");
1571                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1572                         }
1573                         LOG.debug(
1574                                 "update : End processing of the External IPs addition during the update operation");
1575                     }
1576
1577                     //Check if its Update on subnets
1578                     LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1579                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1580                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1581                     Set<Uuid> addedSubnetIds =
1582                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1583                     if (originalSubnetIds != null) {
1584                         addedSubnetIds.removeAll(originalSubnetIds);
1585                     }
1586
1587                     //Check if the Subnet IDs are added during the update.
1588                     if (addedSubnetIds.size() != 0) {
1589                         LOG.debug(
1590                                 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1591                         for (Uuid addedSubnetId : addedSubnetIds) {
1592                 /*
1593                     1) Select the least loaded external IP for the subnet and store the mapping of the
1594                     subnet IP and the external IP in the IntExtIp model.
1595                     2) Increase the count of the selected external IP by one.
1596                     3) Advertise to the BGP if external IP is allocated for the first time for the
1597                     router i.e. the route for the external IP is absent.
1598                  */
1599                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1600                             if (subnetIp != null) {
1601                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1602                                         writeFlowInvTx);
1603                             }
1604                         }
1605                         LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1606                     }
1607
1608                     //Check if the Subnet IDs are removed during the update.
1609                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1610                     removedSubnetIds.removeAll(updatedSubnetIds);
1611                     if (removedSubnetIds.size() != 0) {
1612                         LOG.debug(
1613                                 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1614                         for (Uuid removedSubnetId : removedSubnetIds) {
1615                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1616                             if (subnetAddr != null) {
1617                     /*
1618                         1) Remove the subnet IP and the external IP in the IntExtIp map
1619                         2) Decrease the count of the coresponding external IP by one.
1620                         3) Advertise to the BGP for removing the routes of the corresponding external
1621                         IP if its not allocated to any other internal IP.
1622                     */
1623
1624                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1625                                         subnetAddr[0] + "/" + subnetAddr[1]);
1626                                 if (externalIp == null) {
1627                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1628                                             routerId, subnetAddr[0]);
1629                                     return;
1630                                 }
1631
1632                                 naptManager.updateCounter(routerId, externalIp, false);
1633                                 // Traverse entire model of external-ip counter whether external ip is not
1634                                 // used by any other internal ip in any router
1635                                 if (!isExternalIpAllocated(externalIp)) {
1636                                     LOG.debug("update : external ip is not allocated to any other "
1637                                             + "internal IP so proceeding to remove routes");
1638                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1639                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1640                                             removeFlowInvTx);
1641                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1642                                                     + "router {} with networkId {} and externalIp {}",
1643                                             dpnId, routerId, networkId, externalIp);
1644                                 }
1645
1646                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1647                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1648                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1649                             }
1650                         }
1651                         LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1652                     }
1653                 }));
1654             }));
1655             return futures;
1656         }, NatConstants.NAT_DJC_MAX_RETRIES);
1657     }
1658
1659     private boolean isExternalIpAllocated(String externalIp) {
1660         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1661         Optional<ExternalIpsCounter> externalCountersData;
1662         try {
1663             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1664                         LogicalDatastoreType.OPERATIONAL, id);
1665         } catch (InterruptedException | ExecutionException e) {
1666             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1667             externalCountersData = Optional.empty();
1668         }
1669         if (externalCountersData.isPresent()) {
1670             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1671             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters().values()) {
1672                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter().values()) {
1673                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1674                         if (externalIpCount.getCounter().toJava() != 0) {
1675                             return true;
1676                         }
1677                         break;
1678                     }
1679                 }
1680             }
1681         }
1682         return false;
1683     }
1684
1685     private void allocateExternalIp(Uint64 dpnId, Routers router, Uint32 routerId, String routerName,
1686             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1687         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1688         try {
1689             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1690             if (address instanceof Inet6Address) {
1691                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1692                 return;
1693             }
1694         } catch (UnknownHostException e) {
1695             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1696             return;
1697         }
1698         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1699         if (leastLoadedExtIpAddr != null) {
1700             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1701             String leastLoadedExtIp = externalIpParts[0];
1702             String leastLoadedExtIpPrefix = externalIpParts[1];
1703             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1704             subnetIp = subnetIpParts[0];
1705             String subnetIpPrefix = subnetIpParts[1];
1706             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1707             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1708                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1709                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1710             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1711
1712
1713             // Check if external IP is already assigned a route. (i.e. External IP is previously
1714             // allocated to any of the subnets)
1715             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1716             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1717             Uint32 label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1718             if (label != null) {
1719                 //update
1720                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1721                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1722                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1723                     label, internalIp, leastLoadedExtIpAddrStr);
1724                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1725                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1726                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1727                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1728                 return;
1729             }
1730
1731             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1732             // for the first time and hence not having a route.
1733             //Get the VPN Name using the network ID
1734             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1735             if (vpnName != null) {
1736                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1737                 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1738                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1739                         + "added after gateway-set");
1740                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1741                     if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1742                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1743                         return;
1744                     }
1745                 }
1746                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1747                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1748                     writeFlowInvTx);
1749             }
1750         }
1751     }
1752
1753     @Nullable
1754     protected Uint32 checkExternalIpLabel(Uint32 routerId, String externalIp) {
1755         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1756         for (IpMap ipMap : ipMaps) {
1757             if (ipMap.getExternalIp().equals(externalIp)) {
1758                 if (ipMap.getLabel() != null) {
1759                     return ipMap.getLabel();
1760                 }
1761             }
1762         }
1763         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1764         return null;
1765     }
1766
1767     @Override
1768     public void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1769         if (natMode != NatMode.Controller) {
1770             return;
1771         }
1772         LOG.trace("remove : Router delete method");
1773         /*
1774         ROUTER DELETE SCENARIO
1775         1) Get the router ID from the event.
1776         2) Build the cookie information from the router ID.
1777         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1778         4) Build the flow with the cookie value.
1779         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1780         6) Remove the flows from the other switches which points to the primary and secondary
1781          switches for the flows related the router ID.
1782         7) Get the list of external IP address maintained for the router ID.
1783         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1784         9) Withdraw the corresponding routes from the BGP.
1785          */
1786
1787         if (identifier == null || router == null) {
1788             LOG.error("remove : returning without processing since routers is null");
1789             return;
1790         }
1791
1792         String routerName = router.getRouterName();
1793         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1794             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1795                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1796                         routerName);
1797                 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1798                 if (routerId == NatConstants.INVALID_ID) {
1799                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1800                             routerName);
1801                     return;
1802                 }
1803                 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1804                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1805                 if (bgpVpnUuid != null) {
1806                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1807                 }
1808                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1809                         tx);
1810                 Uuid networkUuid = router.getNetworkId();
1811
1812                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1813                 if (primarySwitchId == null || primarySwitchId.equals(Uint64.ZERO)) {
1814                     // No NAPT switch for external router, probably because the router is not attached to
1815                     // any
1816                     // internal networks
1817                     LOG.debug(
1818                             "No NAPT switch for router {}, check if router is attached to any internal "
1819                                     + "network",
1820                             routerName);
1821                     return;
1822                 } else {
1823                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1824                     final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
1825                     handleDisableSnat(router, networkUuid, externalIps, true, vpnName, primarySwitchId,
1826                             routerId, tx);
1827                 }
1828                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1829                     == NatConstants.INVALID_ID) {
1830                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1831                 }
1832             })), NatConstants.NAT_DJC_MAX_RETRIES);
1833     }
1834
1835     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1836                                   boolean routerFlag, @Nullable String vpnName, Uint64 naptSwitchDpnId,
1837                                   Uint32 routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1838         LOG.info("handleDisableSnat : Entry");
1839         String routerName = router.getRouterName();
1840         try {
1841             if (routerFlag) {
1842                 removeNaptSwitch(routerName);
1843             } else {
1844                 updateNaptSwitch(routerName, Uint64.valueOf(BigInteger.ZERO));
1845             }
1846
1847             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1848             naptManager.removeExternalCounter(routerId);
1849
1850             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1851             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1852                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1853                     + "router ID {} from RouterNaptSwitch model", routerId);
1854                 return;
1855             }
1856             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1857                     networkUuid);
1858             if (extNwProvType == null) {
1859                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1860                 return;
1861             }
1862             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(
1863                     new ArrayList<ExternalIps>(router.getExternalIps().values()));
1864             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1865                     externalSubnetList, removeFlowInvTx, extNwProvType);
1866             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1867             String externalSubnetVpn = null;
1868             for (Uuid externalSubnetId : externalSubnetList) {
1869                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1870                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1871                 if (externalSubnet.isPresent()) {
1872                     externalSubnetVpn =  externalSubnetId.getValue();
1873                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1874                             router.getExtGwMacAddress(), removeFlowInvTx);
1875                 }
1876             }
1877             if (externalSubnetVpn == null) {
1878                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1879                         router.getExtGwMacAddress(), removeFlowInvTx);
1880             }
1881             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1882             // for the router ID.
1883             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1884                     + "router ID {} in the DS", routerId);
1885             naptManager.removeMapping(routerId);
1886         } catch (InterruptedException | ExecutionException e) {
1887             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1888         }
1889         LOG.info("handleDisableSnat : Exit");
1890     }
1891
1892     // TODO Clean up the exception handling
1893     @SuppressWarnings("checkstyle:IllegalCatch")
1894     public void handleDisableSnatInternetVpn(String routerName, Uint32 routerId, Uuid networkUuid,
1895                                              @NonNull Collection<String> externalIps,
1896                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1897         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1898                 + "with internet vpn {}", routerName, vpnId);
1899         try {
1900             Uint64 naptSwitchDpnId = null;
1901             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1902                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1903             Optional<RouterToNaptSwitch> rtrToNapt;
1904             try {
1905                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1906                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1907             } catch (InterruptedException | ExecutionException e) {
1908                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1909                 rtrToNapt = Optional.empty();
1910             }
1911             if (rtrToNapt.isPresent()) {
1912                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1913             }
1914             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1915
1916             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1917                     writeFlowInvTx);
1918             try {
1919                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1920                 if (extGwMacAddress != null) {
1921                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1922                             + "External Router ID {}", extGwMacAddress, routerId);
1923                 } else {
1924                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1925                             + "External Router ID {}", routerId);
1926                     return;
1927                 }
1928                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1929                         writeFlowInvTx);
1930             } catch (Exception ex) {
1931                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1932                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1933             }
1934             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1935                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1936             }
1937         } catch (InterruptedException | ExecutionException e) {
1938             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1939                     + "with internet vpn {}", routerName, vpnId, e);
1940         }
1941         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1942                 routerName, vpnId);
1943     }
1944
1945     // TODO Clean up the exception handling
1946     @SuppressWarnings("checkstyle:IllegalCatch")
1947     public void updateNaptSwitch(String routerName, Uint64 naptSwitchId) {
1948         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1949             .setPrimarySwitchId(naptSwitchId).build();
1950         try {
1951             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1952                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1953         } catch (Exception ex) {
1954             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1955                 naptSwitchId, routerName);
1956         }
1957         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1958             naptSwitchId, routerName);
1959     }
1960
1961     protected void removeNaptSwitch(String routerName) {
1962         // Remove router and switch from model
1963         InstanceIdentifier<RouterToNaptSwitch> id =
1964             InstanceIdentifier.builder(NaptSwitches.class)
1965                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1966         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1967         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1968         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1969         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1970     }
1971
1972     public void removeNaptFlowsFromActiveSwitch(Uint32 routerId, String routerName,
1973                                                 Uint64 dpnId, Uuid networkId, String vpnName,
1974                                                 @NonNull Collection<String> externalIps,
1975                                                 Collection<Uuid> externalSubnetList,
1976                                                 TypedReadWriteTransaction<Configuration> confTx,
1977                                                 ProviderTypes extNwProvType)
1978             throws InterruptedException, ExecutionException {
1979
1980         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1981         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1982
1983         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1984         // traffic which comes from the  VMs of the NAPT switches)
1985         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1986         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1987
1988         LOG.info(
1989             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1990                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1991         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1992
1993         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1994         // traffic which comes from the VMs of the non NAPT switches)
1995         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1996                 elanManager, idManager, routerId, routerName);
1997         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
1998         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1999         LOG.info(
2000             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2001                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
2002         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
2003
2004         //Remove the flow table 25->44 from NAPT Switch
2005         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2006             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
2007         }
2008
2009         //Remove the Outbound flow entry which forwards the packet to FIB Table
2010         LOG.info(
2011             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
2012                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2013
2014         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
2015             NwConstants.IP_PROT_TCP);
2016         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2017             outboundTcpNatFlowRef);
2018         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
2019
2020         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
2021             NwConstants.IP_PROT_UDP);
2022         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2023             outboundUdpNatFlowRef);
2024         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
2025
2026         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
2027             NwConstants.IP_PROT_ICMP);
2028         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2029             icmpDropFlowRef);
2030         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
2031         boolean lastRouterOnExternalNetwork =
2032             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
2033         if (lastRouterOnExternalNetwork) {
2034             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
2035         }
2036         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
2037         // External Subnet Vpn Id.
2038         for (Uuid externalSubnetId : externalSubnetList) {
2039             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2040             if (subnetVpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
2041                 dataBroker, externalSubnetId, routerName, dpnId)) {
2042                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
2043                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
2044                     natPfibSubnetFlowRef);
2045                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2046                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
2047                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
2048                     subnetVpnId, dpnId);
2049             }
2050         }
2051
2052         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
2053         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
2054         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
2055
2056         LOG.info(
2057             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2058             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2059         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2060
2061         if (lastRouterOnExternalNetwork) {
2062             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2063             // - This does not work since ext-routers is deleted already - no network info
2064             //Get the VPN ID from the ExternalNetworks model
2065             Uint32 vpnId = NatConstants.INVALID_ID;
2066             if (vpnName == null || vpnName.isEmpty()) {
2067                 // ie called from router delete cases
2068                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2069                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2070                 if (vpnUuid != null) {
2071                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2072                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2073                               + "or disableSNAT scenario", vpnId, networkId);
2074                 }
2075             } else {
2076                 // ie called from disassociate vpn case
2077                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2078                     vpnName);
2079                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2080                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2081                     vpnId);
2082             }
2083
2084             if (vpnId != NatConstants.INVALID_ID) {
2085                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2086                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2087                 FlowEntity natPfibVpnFlowEntity =
2088                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2089                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2090                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2091                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2092             }
2093         }
2094
2095         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2096         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2097         if (ipPortMapping == null) {
2098             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2099             return;
2100         }
2101
2102         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType().values()) {
2103             String protocol = intextIpProtocolType.getProtocol().name();
2104             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap().values()) {
2105                 String ipPortInternal = ipPortMap.getIpPortInternal();
2106                 String[] ipPortParts = ipPortInternal.split(":");
2107                 if (ipPortParts.length != 2) {
2108                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2109                     return;
2110                 }
2111                 String internalIp = ipPortParts[0];
2112                 String internalPort = ipPortParts[1];
2113
2114                 //Build the flow for the outbound NAPT table
2115                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2116                     + NatConstants.COLON_SEPARATOR + internalPort);
2117                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2118                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2119                 FlowEntity outboundNaptFlowEntity =
2120                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2121
2122                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2123                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2124                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2125
2126                  //Build the flow for the inbound NAPT table
2127                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2128                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2129                 FlowEntity inboundNaptFlowEntity =
2130                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2131
2132                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2133                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2134                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2135             }
2136         }
2137     }
2138
2139     protected void removeNaptFibExternalOutputFlows(Uint32 routerId, Uint64 dpnId, Uuid networkId,
2140                                                     @NonNull Collection<String> externalIps,
2141                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2142             throws ExecutionException, InterruptedException {
2143         Uint32 extVpnId = NatConstants.INVALID_ID;
2144         if (networkId != null) {
2145             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2146             if (vpnUuid != null) {
2147                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2148             } else {
2149                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2150             }
2151         } else {
2152             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2153             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2154         }
2155         if (extVpnId == NatConstants.INVALID_ID) {
2156             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2157             extVpnId = routerId;
2158         }
2159         for (String ip : externalIps) {
2160             String extIp = removeMaskFromIp(ip);
2161             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2162             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2163                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2164                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2165             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2166             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2167         }
2168     }
2169
2170     private String removeMaskFromIp(String ip) {
2171         if (ip != null && !ip.trim().isEmpty()) {
2172             return ip.split("/")[0];
2173         }
2174         return ip;
2175     }
2176
2177     public void removeNaptFlowsFromActiveSwitchInternetVpn(Uint32 routerId, String routerName,
2178                                                            Uint64 dpnId, Uuid networkId, String vpnName,
2179                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2180             throws ExecutionException, InterruptedException {
2181         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2182         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2183
2184         //Remove the NAPT PFIB TABLE entry
2185         Uint32 vpnId = NatConstants.INVALID_ID;
2186         if (vpnName != null) {
2187             // ie called from disassociate vpn case
2188             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2189                     + "with vpnName {}", vpnName);
2190             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2191             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2192                     vpnId);
2193         }
2194
2195         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2196                 networkId, routerName, dpnId)) {
2197             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2198             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2199             FlowEntity natPfibVpnFlowEntity =
2200                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2201             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2202                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2203             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2204
2205             // Remove IP-PORT active NAPT entries and release port from IdManager
2206             // For the router ID get the internal IP , internal port and the corresponding
2207             // external IP and external Port.
2208             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2209             if (ipPortMapping == null) {
2210                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2211                 return;
2212             }
2213             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType().values()) {
2214                 String protocol = intextIpProtocolType.getProtocol().name();
2215                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap().values()) {
2216                     String ipPortInternal = ipPortMap.getIpPortInternal();
2217                     String[] ipPortParts = ipPortInternal.split(":");
2218                     if (ipPortParts.length != 2) {
2219                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2220                                 + "and port");
2221                         return;
2222                     }
2223                     String internalIp = ipPortParts[0];
2224                     String internalPort = ipPortParts[1];
2225
2226                     //Build the flow for the outbound NAPT table
2227                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2228                             + NatConstants.COLON_SEPARATOR + internalPort);
2229                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2230                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2231                     FlowEntity outboundNaptFlowEntity =
2232                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2233
2234                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2235                             + "active switch with the DPN ID {} and router ID {}",
2236                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2237                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2238
2239                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2240                     final String externalIp = ipPortExternal.getIpAddress();
2241
2242                     //Build the flow for the inbound NAPT table
2243                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2244                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2245                     FlowEntity inboundNaptFlowEntity =
2246                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2247
2248                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2249                             + "active active switch with the DPN ID {} and router ID {}",
2250                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2251                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2252
2253                     // Finally release port from idmanager
2254                     String internalIpPort = internalIp + ":" + internalPort;
2255                     naptManager.removePortFromPool(internalIpPort, externalIp);
2256
2257                     //Remove sessions from models
2258                     naptManager.removeIpPortMappingForRouterID(routerId);
2259                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2260                 }
2261             }
2262         } else {
2263             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2264         }
2265     }
2266
2267     public void removeFlowsFromNonActiveSwitches(Uint32 routerId, String routerName,
2268             Uint64 naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2269             throws ExecutionException, InterruptedException {
2270         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2271
2272         // Remove the flows from the other switches which points to the primary and secondary switches
2273         // for the flows related the router ID.
2274         List<Uint64> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2275         if (allSwitchList.isEmpty()) {
2276             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2277             return;
2278         }
2279         for (Uint64 dpnId : allSwitchList) {
2280             if (!naptSwitchDpnId.equals(dpnId)) {
2281                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2282
2283                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2284                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2285                 FlowEntity preSnatFlowEntity =
2286                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2287
2288                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2289                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2290                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2291
2292                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2293                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2294                     NatUtil.getGroupIdKey(routerName));
2295                 if (groupId != NatConstants.INVALID_ID) {
2296                     LOG.info(
2297                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2298                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2299                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId.longValue());
2300                 } else {
2301                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2302                 }
2303             }
2304         }
2305     }
2306
2307     public void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, @Nullable Uuid networkUuid,
2308                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2309                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2310             throws ExecutionException, InterruptedException {
2311         //Withdraw the corresponding routes from the BGP.
2312         //Get the network ID using the router ID.
2313         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2314                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2315         if (networkUuid == null) {
2316             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2317             return;
2318         }
2319
2320         if (externalIps.isEmpty()) {
2321             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2322             return;
2323         }
2324
2325         if (vpnName == null) {
2326             //Get the VPN Name using the network ID
2327             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2328             if (vpnName == null) {
2329                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2330                     networkUuid, routerId);
2331                 return;
2332             }
2333         }
2334         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2335
2336         //Remove custom FIB routes
2337         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2338         for (String extIp : externalIps) {
2339             extIp = NatUtil.validateAndAddNetworkMask(extIp);
2340             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2341         }
2342     }
2343
2344     protected void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2345                                             final Uuid networkUuid, String extGwMacAddress,
2346                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2347             throws ExecutionException, InterruptedException {
2348         clearBgpRoutes(extIp, vpnName);
2349         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2350                 removeFlowInvTx);
2351     }
2352
2353     protected void delFibTsAndReverseTraffic(final Uint64 dpnId, String routerName, Uint32 routerId, String extIp,
2354                                              String vpnName, Uuid extNetworkId, Uint32 tempLabel,
2355                                              String gwMacAddress, boolean switchOver,
2356                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2357             throws ExecutionException, InterruptedException {
2358         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2359         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2360         if (routerName == null) {
2361             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2362             return;
2363         }
2364         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2365         if (extNwProvType == null) {
2366             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2367             return;
2368         }
2369         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2370          * external network provided type is VxLAN
2371          */
2372         if (extNwProvType == ProviderTypes.VXLAN) {
2373             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2374             );
2375             return;
2376         }
2377         if (tempLabel.longValue() < 0) {
2378             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2379                     extIp, routerId);
2380             return;
2381         }
2382         final Uint32 label = tempLabel;
2383         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2384         RemoveFibEntryInput input = null;
2385         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2386             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2387             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2388             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2389                     extRouter);
2390
2391             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2392                     externalSubnetId);
2393
2394             if (externalSubnet.isPresent()) {
2395                 vpnName =  externalSubnetId.getValue();
2396             }
2397         }
2398         final String externalVpn = vpnName;
2399         if (label != null && label.toJava() <= 0) {
2400             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2401                 extIp, routerId);
2402             input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2403                 .setSourceDpid(dpnId).setIpAddress(externalIp)
2404                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2405         } else {
2406             input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2407                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2408                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2409             removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2410             removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2411         }
2412         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2413
2414         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2415         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2416         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2417             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2418             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2419         }
2420         if (!switchOver) {
2421             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2422                 Futures.transformAsync(future, result -> {
2423                     //Release label
2424                     if (result.isSuccessful() && label != null && label.toJava() > 0) {
2425                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2426                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2427                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2428                         Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
2429                         if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
2430                             String errMsg = String.format(
2431                                     "ExternalRoutersListener: RPC call to remove VPN label "
2432                                             + "on dpn %s for prefix %s failed for vpn %s - %s",
2433                                     dpnId, externalIp, result.getErrors());
2434                             LOG.error(errMsg);
2435                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2436                         }
2437                         return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2438                     } else {
2439                         String errMsg =
2440                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2441                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2442                         LOG.error(errMsg);
2443                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2444                     }
2445                 }, MoreExecutors.directExecutor());
2446
2447             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2448
2449                 @Override
2450                 public void onFailure(@NonNull Throwable error) {
2451                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2452                         + "got external ip {}", label, extIp, error);
2453                 }
2454
2455                 @Override
2456                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2457                     if (result.isSuccessful()) {
2458                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2459                             + "from VPN {}", externalIp, externalVpn);
2460                     } else {
2461                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2462                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2463                     }
2464                 }
2465             }, MoreExecutors.directExecutor());
2466         } else {
2467             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2468                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2469         }
2470     }
2471
2472     private void delFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2473                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2474                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2475             throws ExecutionException, InterruptedException {
2476         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2477         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2478         if (routerName == null) {
2479             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2480             return;
2481         }
2482         //Get the external network provider type from networkId
2483         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2484         if (extNwProvType == null) {
2485             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2486                     networkUuid);
2487             return;
2488         }
2489         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2490          *  external network provided type is VxLAN
2491          */
2492         if (extNwProvType == ProviderTypes.VXLAN) {
2493             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2494             return;
2495         }
2496         //Get IPMaps from the DB for the router ID
2497         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2498         if (dbIpMaps.isEmpty()) {
2499             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2500             return;
2501         }
2502
2503         Uint32 tempLabel = NatConstants.INVALID_ID;
2504         for (IpMap dbIpMap : dbIpMaps) {
2505             String dbExternalIp = dbIpMap.getExternalIp();
2506             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2507             //Select the IPMap, whose external IP is the IP for which FIB is installed
2508             if (extIp.equals(dbExternalIp)) {
2509                 tempLabel = dbIpMap.getLabel();
2510                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2511                     tempLabel, dbExternalIp, routerId);
2512                 break;
2513             }
2514         }
2515         if (tempLabel == NatConstants.INVALID_ID) {
2516             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2517                     extIp, routerId);
2518             return;
2519         }
2520
2521         final Uint32 label = tempLabel;
2522         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2523         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2524                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2525                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2526         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2527
2528         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2529         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2530         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2531             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2532             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2533         }
2534         if (!switchOver) {
2535             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2536                     Futures.transformAsync(future, result -> {
2537                         //Release label
2538                         if (result.isSuccessful()) {
2539                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2540                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2541                             Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService
2542                                     .removeVpnLabel(labelInput);
2543                             if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
2544                                 String errMsg = String.format(
2545                                         "RPC call to remove VPN label on dpn %s for prefix %s "
2546                                                 + "failed for vpn %s - %s", dpnId, externalIp, vpnName,
2547                                         result.getErrors());
2548                                 LOG.error(errMsg);
2549                                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2550                             }
2551                             return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2552                         } else {
2553                             String errMsg =
2554                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2555                                             + "prefix %s Failed - %s",
2556                                             dpnId, externalIp, result.getErrors());
2557                             LOG.error(errMsg);
2558                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2559                         }
2560                     }, MoreExecutors.directExecutor());
2561
2562             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2563
2564                 @Override
2565                 public void onFailure(@NonNull Throwable error) {
2566                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2567                 }
2568
2569                 @Override
2570                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2571                     if (result.isSuccessful()) {
2572                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2573                             + "from VPN {}", externalIp, vpnName);
2574                     } else {
2575                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2576                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2577                     }
2578                 }
2579             }, MoreExecutors.directExecutor());
2580         } else {
2581             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2582                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2583         }
2584     }
2585
2586     protected void clearFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, Uuid networkUuid,
2587             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2588             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2589         //Withdraw the corresponding routes from the BGP.
2590         //Get the network ID using the router ID.
2591         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2592                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2593         if (networkUuid == null) {
2594             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2595             return;
2596         }
2597
2598         if (externalIps == null || externalIps.isEmpty()) {
2599             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2600             return;
2601         }
2602
2603         if (vpnName == null) {
2604             //Get the VPN Name using the network ID
2605             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2606             if (vpnName == null) {
2607                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2608                     networkUuid, routerId);
2609                 return;
2610             }
2611         }
2612         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2613
2614         //Remove custom FIB routes
2615         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2616         for (String extIp : externalIps) {
2617             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2618                     writeFlowInvTx);
2619         }
2620     }
2621
2622     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2623         //Inform BGP about the route removal
2624         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2625         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2626         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName);
2627         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), externalIp);
2628     }
2629
2630     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
2631             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2632         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2633         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2634             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2635         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2636     }
2637
2638     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
2639             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2640         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2641         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2642         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2643         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2644     }
2645
2646     /**
2647      * router association to vpn.
2648      *
2649      * @param routerName - Name of router
2650      * @param routerId - router id
2651      * @param bgpVpnName BGP VPN name
2652      */
2653     public void changeLocalVpnIdToBgpVpnId(String routerName, Uint32 routerId, String extNetwork, String bgpVpnName,
2654             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2655         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2656         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2657             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2658
2659             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2660
2661             if (bgpVpnId != NatConstants.INVALID_ID) {
2662                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2663                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2664                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2665                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2666                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2667                     getRoutersIdentifier(bgpVpnId), rtrs);
2668
2669                 // Get the allocated Primary NAPT Switch for this router
2670                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2671                         routerId, bgpVpnId);
2672                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2673
2674                 // Get the group ID
2675                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2676                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, new Uuid(extNetwork),
2677                     true, writeFlowInvTx, extNwProvType);
2678             }
2679         }
2680     }
2681
2682     /**
2683      * router disassociation from vpn.
2684      *
2685      * @param routerName - Name of router
2686      * @param routerId - router id
2687      * @param bgpVpnName BGP VPN name
2688      */
2689     public void changeBgpVpnIdToLocalVpnId(String routerName, Uint32 routerId, String bgpVpnName, String extNetwork,
2690             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2691         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2692         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2693             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2694             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2695
2696             // Get the allocated Primary NAPT Switch for this router
2697             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2698
2699             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2700             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2701
2702             // Get the group ID
2703             Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2704             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId,
2705                 new Uuid(extNetwork), true, writeFlowInvTx, extNwProvType);
2706         }
2707     }
2708
2709     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2710         InstanceIdentifier<Routers> routerInstanceIndentifier =
2711             InstanceIdentifier.builder(ExtRouters.class)
2712                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2713         try {
2714             Optional<Routers> routerData = SingleTransactionDataBroker
2715                     .syncReadOptional(dataBroker,
2716                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2717             return routerData.isPresent() && routerData.get().isEnableSnat();
2718         } catch (InterruptedException | ExecutionException e) {
2719             LOG.error("Failed to read data for router id {}", routerUuid, e);
2720             return false;
2721         }
2722     }
2723
2724     public void installFlowsWithUpdatedVpnId(Uint64 primarySwitchId, String routerName, Uint32 bgpVpnId,
2725                                              Uint32 routerId, Uuid extNwUuid, boolean isSnatCfgd,
2726                                              TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2727
2728         Uint32 changedVpnId = bgpVpnId;
2729         String idType = "BGP VPN";
2730         if (bgpVpnId == NatConstants.INVALID_ID) {
2731             changedVpnId = routerId;
2732             idType = "router";
2733         }
2734
2735         List<Uint64> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2736         if (switches.isEmpty()) {
2737             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2738             return;
2739         }
2740         for (Uint64 dpnId : switches) {
2741             // Update the BGP VPN ID in the SNAT miss entry to group
2742             if (!dpnId.equals(primarySwitchId)) {
2743                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2744                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2745                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2746                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2747                     NatUtil.getGroupIdKey(routerName));
2748                 if (groupId != NatConstants.INVALID_ID) {
2749                     if (!isSnatCfgd) {
2750                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2751                     }
2752                     LOG.debug(
2753                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2754                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2755                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2756                         groupId, changedVpnId);
2757                     mdsalManager.addFlow(confTx, flowEntity);
2758                 } else {
2759                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2760                 }
2761             } else {
2762                 LOG.debug(
2763                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2764                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2765                 FlowEntity flowEntity =
2766                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2767                 mdsalManager.addFlow(confTx, flowEntity);
2768
2769                 LOG.debug(
2770                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2771                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2772                         idType, changedVpnId, primarySwitchId);
2773                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2774                         changedVpnId, confTx, extNwProvType);
2775
2776                 LOG.debug(
2777                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2778                                 + "which punts the packet to the controller in the Primary switch {}",
2779                         idType, changedVpnId, primarySwitchId);
2780                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2781
2782                 LOG.debug(
2783                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2784                                 + " outgoing packet to FIB Table in the Primary switch {}",
2785                         idType, changedVpnId, primarySwitchId);
2786                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2787
2788                 LOG.debug(
2789                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2790                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2791                                 + " {}", idType, changedVpnId, primarySwitchId);
2792                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2793
2794                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2795                         primarySwitchId);
2796                 //Get the VPN ID from the ExternalNetworks model
2797                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, extNwUuid);
2798                 if (vpnUuid != null) {
2799                     Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2800                     //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table
2801                     // matching on the VPN ID.
2802                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
2803                         installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2804                     }
2805                 } else {
2806                     LOG.error("NAT Service : vpnUuid is null");
2807                 }
2808             }
2809         }
2810     }
2811
2812     public void updateNaptFlowsWithVpnId(Uint64 dpnId, String routerName, Uint32 routerId, Uint32 bgpVpnId) {
2813         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2814         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2815         if (ipPortMapping == null) {
2816             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2817             return;
2818         }
2819         // Get the External Gateway MAC Address
2820         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2821         if (extGwMacAddress != null) {
2822             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2823                     extGwMacAddress, routerId);
2824         } else {
2825             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2826                     routerId);
2827             return;
2828         }
2829         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType().values()) {
2830             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap().values()) {
2831                 String ipPortInternal = ipPortMap.getIpPortInternal();
2832                 String[] ipPortParts = ipPortInternal.split(":");
2833                 if (ipPortParts.length != 2) {
2834                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2835                     return;
2836                 }
2837                 String internalIp = ipPortParts[0];
2838                 String internalPort = ipPortParts[1];
2839                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2840                         internalIp, internalPort);
2841                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2842                 NAPTEntryEvent.Protocol protocol;
2843                 switch (protocolTypes) {
2844                     case TCP:
2845                         protocol = NAPTEntryEvent.Protocol.TCP;
2846                         break;
2847                     case UDP:
2848                         protocol = NAPTEntryEvent.Protocol.UDP;
2849                         break;
2850                     default:
2851                         protocol = NAPTEntryEvent.Protocol.TCP;
2852                 }
2853                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2854                 SessionAddress externalAddress =
2855                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2856                 Uint32 internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2857                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2858                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2859                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2860                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2861             }
2862         }
2863     }
2864
2865     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName, Uint32 groupId,
2866                                                           Uint32 changedVpnId) {
2867
2868         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2869             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2870         List<MatchInfo> matches = new ArrayList<>();
2871         matches.add(MatchEthernetType.IPV4);
2872         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2873                 MetaDataUtil.METADATA_MASK_VRFID));
2874
2875         List<ActionInfo> actionsInfo = new ArrayList<>();
2876         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2877                 elanManager, idManager, changedVpnId, routerName);
2878         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
2879         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2880                 actionsInfo);
2881         actionsInfo.add(new ActionGroup(groupId.longValue()));
2882         List<InstructionInfo> instructions = new ArrayList<>();
2883         instructions.add(new InstructionApplyActions(actionsInfo));
2884         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2885         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2886             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2887             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2888
2889         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2890         return flowEntity;
2891     }
2892
2893     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(Uint64 dpId, String routerName,
2894                                                                         Uint32 changedVpnId) {
2895
2896         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2897                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2898         List<MatchInfo> matches = new ArrayList<>();
2899         matches.add(MatchEthernetType.IPV4);
2900         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2901                 MetaDataUtil.METADATA_MASK_VRFID));
2902
2903         List<InstructionInfo> instructions = new ArrayList<>();
2904         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2905
2906         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2907         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2908             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2909             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2910
2911         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2912         return flowEntity;
2913     }
2914
2915     // TODO : Replace this with ITM Rpc once its available with full functionality
2916     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(Uint64 dpnId, String routerName,
2917                                                                      Uint32 routerId, Uint32 changedVpnId,
2918                                                                      TypedWriteTransaction<Configuration> confTx,
2919                                                                      ProviderTypes extNwProvType) {
2920
2921         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2922             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2923         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2924                 extNwProvType);
2925         mdsalManager.addFlow(confTx, flowEntity);
2926     }
2927
2928     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName,
2929                                                          Uint32 routerIdLongVal, Uint32 changedVpnId,
2930                                                          ProviderTypes extNwProvType) {
2931         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2932             dpId, routerName, changedVpnId);
2933         List<MatchInfo> matches = new ArrayList<>();
2934         matches.add(MatchEthernetType.IPV4);
2935
2936         Uint64 tunnelId = Uint64.valueOf(changedVpnId);
2937         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2938             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2939         }
2940         matches.add(new MatchTunnelId(tunnelId));
2941
2942         List<InstructionInfo> instructions = new ArrayList<>();
2943         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2944             MetaDataUtil.METADATA_MASK_VRFID));
2945         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2946         Uint32 routerId = routerIdLongVal;
2947         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
2948         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2949             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2950             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2951         return flowEntity;
2952     }
2953
2954     public void createOutboundTblEntryWithBgpVpn(Uint64 dpnId, Uint32 routerId, Uint32 changedVpnId,
2955                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2956         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2957             dpnId, routerId, changedVpnId);
2958         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2959                 NwConstants.IP_PROT_TCP);
2960         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2961         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2962
2963         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2964                 NwConstants.IP_PROT_UDP);
2965         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2966         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2967
2968         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2969         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2970         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2971     }
2972
2973     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(Uint64 dpId, Uint32 routerId,
2974                                                            Uint32 changedVpnId, int protocol) {
2975         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2976             dpId, routerId, changedVpnId);
2977         Uint64 cookie = getCookieOutboundFlow(routerId);
2978         List<MatchInfo> matches = new ArrayList<>();
2979         matches.add(MatchEthernetType.IPV4);
2980         matches.add(new MatchIpProtocol((short)protocol));
2981         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2982                 MetaDataUtil.METADATA_MASK_VRFID));
2983
2984         List<InstructionInfo> instructions = new ArrayList<>();
2985         List<ActionInfo> actionsInfos = new ArrayList<>();
2986         actionsInfos.add(new ActionPuntToController());
2987         if (snatPuntTimeout != 0) {
2988             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2989         }
2990         instructions.add(new InstructionApplyActions(actionsInfos));
2991
2992         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2993         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2994             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2995         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2996         return flowEntity;
2997     }
2998
2999     public void installNaptPfibEntryWithBgpVpn(Uint64 dpnId, Uint32 segmentId, Uint32 changedVpnId,
3000                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
3001         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
3002             dpnId, segmentId, changedVpnId);
3003         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
3004         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
3005     }
3006
3007     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(Uint64 dpId, Uint32 segmentId, Uint32 changedVpnId) {
3008
3009         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
3010             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
3011         List<MatchInfo> matches = new ArrayList<>();
3012         matches.add(MatchEthernetType.IPV4);
3013         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
3014                 MetaDataUtil.METADATA_MASK_VRFID));
3015
3016         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
3017         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
3018         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
3019         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
3020         instructionInfo.add(new InstructionApplyActions(listActionInfo));
3021
3022         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
3023         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
3024             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
3025             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
3026         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
3027         return flowEntity;
3028     }
3029
3030     protected void installNaptPfibEntriesForExternalSubnets(String routerName, Uint64 dpnId,
3031                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
3032         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
3033                 routerName);
3034         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
3035             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
3036             if (subnetVpnId != NatConstants.INVALID_ID) {
3037                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
3038                     + "and vpnId {}", dpnId, subnetVpnId);
3039                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
3040             }
3041         }
3042     }
3043 }