natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.JdkFutureAdapters;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19
  20 import java.math.BigInteger;
  21 import java.net.Inet6Address;
  22 import java.net.InetAddress;
  23 import java.net.UnknownHostException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Collection;
  27 import java.util.Collections;
  28 import java.util.HashMap;
  29 import java.util.HashSet;
  30 import java.util.List;
  31 import java.util.Map;
  32 import java.util.Objects;
  33 import java.util.Set;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.concurrent.Future;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.NonNull;
  40 import org.eclipse.jdt.annotation.Nullable;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  44 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  45 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  46 import org.opendaylight.genius.infra.Datastore.Configuration;
  47 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  48 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  49 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  50 import org.opendaylight.genius.infra.TypedWriteTransaction;
  51 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  52 import org.opendaylight.genius.mdsalutil.ActionInfo;
  53 import org.opendaylight.genius.mdsalutil.BucketInfo;
  54 import org.opendaylight.genius.mdsalutil.FlowEntity;
  55 import org.opendaylight.genius.mdsalutil.GroupEntity;
  56 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  57 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  58 import org.opendaylight.genius.mdsalutil.MatchInfo;
  59 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  60 import org.opendaylight.genius.mdsalutil.NwConstants;
  61 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  70 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  71 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  73 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  74 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  75 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  79 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  80 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  81 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  82 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  83 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  84 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  85 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  86 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 147 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 148 import org.opendaylight.yangtools.yang.common.RpcResult;
 149 import org.opendaylight.yangtools.yang.common.Uint32;
 150 import org.opendaylight.yangtools.yang.common.Uint64;
 151 import org.slf4j.Logger;
 152 import org.slf4j.LoggerFactory;
 153
 154
 155 @Singleton
 156 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 157     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 158
 159     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
 160     private static final Uint64 COOKIE_VM_LFIB_TABLE = Uint64.valueOf("8000022", 16).intern();
 161
 162     private final DataBroker dataBroker;
 163     private final ManagedNewTransactionRunner txRunner;
 164     private final IMdsalApiManager mdsalManager;
 165     private final ItmRpcService itmManager;
 166     private final OdlInterfaceRpcService odlInterfaceRpcService;
 167     private final IdManagerService idManager;
 168     private final NaptManager naptManager;
 169     private final NAPTSwitchSelector naptSwitchSelector;
 170     private final IBgpManager bgpManager;
 171     private final VpnRpcService vpnService;
 172     private final FibRpcService fibService;
 173     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 174     private final NaptEventHandler naptEventHandler;
 175     private final NaptPacketInHandler naptPacketInHandler;
 176     private final IFibManager fibManager;
 177     private final IVpnManager vpnManager;
 178     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 179     private final NatMode natMode;
 180     private final IElanService elanManager;
 181     private final JobCoordinator coordinator;
 182     private final IInterfaceManager interfaceManager;
 183     private final NatOverVxlanUtil natOverVxlanUtil;
 184     private final int snatPuntTimeout;
 185
 186     @Inject
 187     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 188                                    final ItmRpcService itmManager,
 189                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 190                                    final IdManagerService idManager,
 191                                    final NaptManager naptManager,
 192                                    final NAPTSwitchSelector naptSwitchSelector,
 193                                    final IBgpManager bgpManager,
 194                                    final VpnRpcService vpnService,
 195                                    final FibRpcService fibService,
 196                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 197                                    final NaptEventHandler naptEventHandler,
 198                                    final NaptPacketInHandler naptPacketInHandler,
 199                                    final IFibManager fibManager,
 200                                    final IVpnManager vpnManager,
 201                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 202                                    final NatserviceConfig config,
 203                                    final IElanService elanManager,
 204                                    final JobCoordinator coordinator,
 205                                    final NatOverVxlanUtil natOverVxlanUtil,
 206                                    final IInterfaceManager interfaceManager) {
 207         super(Routers.class, ExternalRoutersListener.class);
 208         this.dataBroker = dataBroker;
 209         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 210         this.mdsalManager = mdsalManager;
 211         this.itmManager = itmManager;
 212         this.odlInterfaceRpcService = odlInterfaceRpcService;
 213         this.idManager = idManager;
 214         this.naptManager = naptManager;
 215         this.naptSwitchSelector = naptSwitchSelector;
 216         this.bgpManager = bgpManager;
 217         this.vpnService = vpnService;
 218         this.fibService = fibService;
 219         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 220         this.naptEventHandler = naptEventHandler;
 221         this.naptPacketInHandler = naptPacketInHandler;
 222         this.fibManager = fibManager;
 223         this.vpnManager = vpnManager;
 224         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 225         this.elanManager = elanManager;
 226         this.coordinator = coordinator;
 227         this.interfaceManager = interfaceManager;
 228         this.natOverVxlanUtil = natOverVxlanUtil;
 229         if (config != null) {
 230             this.natMode = config.getNatMode();
 231             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 232         } else {
 233             this.natMode = NatMode.Controller;
 234             this.snatPuntTimeout = 0;
 235         }
 236     }
 237
 238     @Override
 239     @PostConstruct
 240     public void init() {
 241         LOG.info("{} init", getClass().getSimpleName());
 242         // This class handles ExternalRouters for Controller SNAT mode.
 243         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 244         if (natMode == NatMode.Controller) {
 245             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 246             NatUtil.createGroupIdPool(idManager);
 247         }
 248     }
 249
 250     @Override
 251     protected InstanceIdentifier<Routers> getWildCardPath() {
 252         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 253     }
 254
 255     @Override
 256     // TODO Clean up the exception handling
 257     @SuppressWarnings("checkstyle:IllegalCatch")
 258     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 259         // Populate the router-id-name container
 260         String routerName = routers.getRouterName();
 261         LOG.info("add : external router event for {}", routerName);
 262         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
 263         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 264         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 265         try {
 266             if (routers.isEnableSnat()) {
 267                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 268                     () -> Collections.singletonList(
 269                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 270                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 271                             Uint32 bgpVpnId = NatConstants.INVALID_ID;
 272                             if (bgpVpnUuid != null) {
 273                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 274                             }
 275                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 276                             // Allocate Primary Napt Switch for this router
 277                             Uint64 primarySwitchId = getPrimaryNaptSwitch(routerName);
 278                             if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 279                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 280                             }
 281                         }
 282                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 283             } else {
 284                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 285             }
 286         } catch (Exception ex) {
 287             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 288                     routerName, ex);
 289         }
 290     }
 291
 292     public void handleEnableSnat(Routers routers, Uint32 routerId, Uint64 primarySwitchId, Uint32 bgpVpnId,
 293                                  TypedWriteTransaction<Configuration> confTx) {
 294         String routerName = routers.getRouterName();
 295         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 296
 297         naptManager.initialiseExternalCounter(routers, routerId);
 298         subnetRegisterMapping(routers, routerId);
 299
 300         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 301             primarySwitchId, routerName);
 302
 303         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 304                 routers.getNetworkId());
 305         if (extNwProvType == null) {
 306             LOG.error("handleEnableSnat : External Network Provider Type missing");
 307             return;
 308         }
 309
 310         if (bgpVpnId != NatConstants.INVALID_ID) {
 311             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId,
 312                 routers.getNetworkId(),false, confTx, extNwProvType);
 313         } else {
 314             // write metadata and punt
 315             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 316             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, routers.getNetworkId(), confTx);
 317             // Now install entries in SNAT tables to point to Primary for each router
 318             List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 319             for (Uint64 dpnId : switches) {
 320                 // Handle switches and NAPT switches separately
 321                 if (!dpnId.equals(primarySwitchId)) {
 322                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 323                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 324                 }
 325             }
 326         }
 327
 328         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 329         if (externalIps.isEmpty()) {
 330             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 331             return;
 332         } else {
 333             for (String externalIpAddrPrefix : externalIps) {
 334                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 335                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 336                 externalIpAddrPrefix = NatUtil.validateAndAddNetworkMask(externalIpAddrPrefix);
 337                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 338                 );
 339             }
 340         }
 341         LOG.debug("handleEnableSnat : Exit");
 342     }
 343
 344     private Uint64 getPrimaryNaptSwitch(String routerName) {
 345         // Allocate Primary Napt Switch for this router
 346         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 347         if (primarySwitchId != null && !primarySwitchId.equals(Uint64.ZERO)) {
 348             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 349                 primarySwitchId, routerName);
 350             return primarySwitchId;
 351         }
 352         return selectNewNAPTSwitch(routerName);
 353     }
 354
 355     private Uint64 selectNewNAPTSwitch(String routerName) {
 356         // Allocated an id from VNI pool for the Router.
 357         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 358         Uint64 primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName, null);
 359         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 360
 361         return primarySwitchId;
 362     }
 363
 364     protected void installNaptPfibExternalOutputFlow(String routerName, Uint32 routerId, Uint64 dpnId,
 365                                                      TypedWriteTransaction<Configuration> confTx) {
 366         Uint32 extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 367         if (extVpnId == NatConstants.INVALID_ID) {
 368             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 369             extVpnId = routerId;
 370         }
 371         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 372         if (externalIps.isEmpty()) {
 373             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 374                 dpnId, extVpnId);
 375             return;
 376         }
 377         for (String ip : externalIps) {
 378             Uuid subnetId = getSubnetIdForFixedIp(ip);
 379             if (subnetId != null) {
 380                 Uint32 subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 381                 if (subnetVpnId != NatConstants.INVALID_ID) {
 382                     extVpnId = subnetVpnId;
 383                 }
 384                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 385                     dpnId, extVpnId, subnetId);
 386                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 387                 if (postNaptFlowEntity != null) {
 388                     mdsalManager.addFlow(confTx, postNaptFlowEntity);
 389                 }
 390             }
 391         }
 392     }
 393
 394     @Nullable
 395     private Uuid getSubnetIdForFixedIp(String ip) {
 396         if (ip != null) {
 397             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 398             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 399             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 400         }
 401         LOG.error("getSubnetIdForFixedIp : ip is null");
 402         return null;
 403     }
 404
 405     protected void subnetRegisterMapping(Routers routerEntry, Uint32 segmentId) {
 406         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 407         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 408         int counter = 0;
 409         int extIpCounter = externalIps.size();
 410         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 411                 counter, extIpCounter);
 412         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 413         if (subnetIds == null) {
 414             return;
 415         }
 416         for (Uuid subnet : subnetIds) {
 417             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 418             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 419                 .builder(Subnetmaps.class)
 420                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 421                 .build();
 422             Optional<Subnetmap> sn;
 423             try {
 424                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 425                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 426             } catch (ReadFailedException e) {
 427                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 428                 sn = Optional.absent();
 429             }
 430             if (sn.isPresent()) {
 431                 // subnets
 432                 Subnetmap subnetmapEntry = sn.get();
 433                 String subnetString = subnetmapEntry.getSubnetIp();
 434                 String[] subnetSplit = subnetString.split("/");
 435                 String subnetIp = subnetSplit[0];
 436                 try {
 437                     InetAddress address = InetAddress.getByName(subnetIp);
 438                     if (address instanceof Inet6Address) {
 439                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 440                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 441                         continue;
 442                     }
 443                 } catch (UnknownHostException e) {
 444                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 445                     return;
 446                 }
 447                 String subnetPrefix = "0";
 448                 if (subnetSplit.length == 2) {
 449                     subnetPrefix = subnetSplit[1];
 450                 }
 451                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 452                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 453                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 454                 //externalIps
 455                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 456                         counter, extIpCounter);
 457                 if (extIpCounter != 0) {
 458                     if (counter < extIpCounter) {
 459                         String[] ipSplit = externalIps.get(counter).split("/");
 460                         String externalIp = ipSplit[0];
 461                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 462                         if (ipSplit.length == 2) {
 463                             extPrefix = ipSplit[1];
 464                         }
 465                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 466                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 467                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 468                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 469                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 470                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 471                     } else {
 472                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 473                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 474                         String[] ipSplit = externalIps.get(counter).split("/");
 475                         String externalIp = ipSplit[0];
 476                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 477                         if (ipSplit.length == 2) {
 478                             extPrefix = ipSplit[1];
 479                         }
 480                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 481                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 482                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 483                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 484                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 485                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 486                             externalIp, extPrefix);
 487                     }
 488                 }
 489                 counter++;
 490                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 491             } else {
 492                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 493             }
 494         }
 495     }
 496
 497     private void addOrDelDefFibRouteToSNAT(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 498             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 499             throws ExecutionException, InterruptedException {
 500         //Check if BGP VPN exists. If exists then invoke the new method.
 501         if (bgpVpnId != NatConstants.INVALID_ID) {
 502             if (bgpVpnUuid != null) {
 503                 String bgpVpnName = bgpVpnUuid.getValue();
 504                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 505                     bgpVpnId, bgpVpnName);
 506                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 507                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 508                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 509             }
 510             if (create) {
 511                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 512             } else {
 513                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 514             }
 515             return;
 516         }
 517
 518         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 519         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 520     }
 521
 522     private void addOrDelDefaultFibRouteForSNAT(String routerName, Uint32 routerId, boolean create,
 523             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 524         List<Uint64> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 525         if (switches.isEmpty()) {
 526             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 527             return;
 528         }
 529         if (routerId == NatConstants.INVALID_ID) {
 530             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 531                     + "default NAT route in FIB", routerName);
 532             return;
 533         }
 534         for (Uint64 dpnId : switches) {
 535             if (create) {
 536                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 537                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 538                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 539             } else {
 540                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 541                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 542                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 543             }
 544         }
 545     }
 546
 547     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId,
 548                                                      Uint32 bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 549         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 550         if (dpnIds.isEmpty()) {
 551             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 552                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 553             return;
 554         }
 555         for (Uint64 dpnId : dpnIds) {
 556             if (bgpVpnId != NatConstants.INVALID_ID) {
 557                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 558                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 559                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 560             } else {
 561                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 562                     + "in dpn {} for the internal vpn", routerId, dpnId);
 563                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 564             }
 565         }
 566     }
 567
 568     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, Uint32 routerId, Uint32 bgpVpnId,
 569                                                         TypedReadWriteTransaction<Configuration> confTx)
 570             throws ExecutionException, InterruptedException {
 571         List<Uint64> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 572         if (dpnIds.isEmpty()) {
 573             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 574                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 575             return;
 576         }
 577         for (Uint64 dpnId : dpnIds) {
 578             if (bgpVpnId != NatConstants.INVALID_ID) {
 579                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 580                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 581                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 582             } else {
 583                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 584                     + "in dpn {} for the internal vpn", routerId, dpnId);
 585                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 586             }
 587         }
 588     }
 589
 590     protected void installOutboundMissEntry(String routerName, Uint32 routerId, Uint64 primarySwitchId,
 591         TypedWriteTransaction<Configuration> confTx) {
 592         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 593         if (routerId != NatConstants.INVALID_ID) {
 594             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 595                     primarySwitchId, routerId);
 596             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 597         } else {
 598             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 599                 + "createAndInstallMissEntry", routerName);
 600         }
 601     }
 602
 603     public String getFlowRefOutbound(Uint64 dpnId, short tableId, Uint32 routerID, int protocol) {
 604         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 605                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 606     }
 607
 608     private String getFlowRefNaptPreFib(Uint64 dpnId, short tableId, Uint32 vpnId) {
 609         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 610                 .FLOWID_SEPARATOR + vpnId;
 611     }
 612
 613     public Uint64 getCookieOutboundFlow(Uint32 routerId) {
 614         return Uint64.valueOf((NwConstants.COOKIE_OUTBOUND_NAPT_TABLE).toJava().add(
 615                 new BigInteger("0110001", 16)).add(BigInteger.valueOf(routerId.longValue())));
 616     }
 617
 618     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, Uint64 cookie) {
 619         long l4SrcPortField;
 620         long l4DstPortField;
 621         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 622
 623         if (protocol == NwConstants.IP_PROT_TCP) {
 624             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 625             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 626         } else {
 627             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 628             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 629         }
 630         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 631                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 632                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 633                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 634                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 635                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 636                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 637                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 638                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 639                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 640                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 641                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 642                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 643                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 644                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 645
 646         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 647                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 648     }
 649
 650     private FlowEntity buildIcmpDropFlow(Uint64 dpnId, Uint32 routerId, Uint32 vpnId) {
 651         List<MatchInfo> matches = new ArrayList<>();
 652         matches.add(MatchEthernetType.IPV4);
 653         matches.add(MatchIpProtocol.ICMP);
 654         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId.longValue()),
 655                 MetaDataUtil.METADATA_MASK_VRFID));
 656
 657         List<ActionInfo> actionInfos = new ArrayList<>();
 658         actionInfos.add(new ActionDrop());
 659
 660         List<InstructionInfo> instructions = new ArrayList<>();
 661         instructions.add(new InstructionApplyActions(actionInfos));
 662
 663         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 664                 NwConstants.IP_PROT_ICMP);
 665         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 666                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 667                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 668         return flow;
 669     }
 670
 671     protected FlowEntity buildOutboundFlowEntity(Uint64 dpId, Uint32 routerId, int protocol) {
 672         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 673         Uint64 cookie = getCookieOutboundFlow(routerId);
 674         List<MatchInfo> matches = new ArrayList<>();
 675         matches.add(MatchEthernetType.IPV4);
 676         matches.add(new MatchIpProtocol((short)protocol));
 677         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 678                 MetaDataUtil.METADATA_MASK_VRFID));
 679
 680         List<InstructionInfo> instructions = new ArrayList<>();
 681         List<ActionInfo> actionsInfos = new ArrayList<>();
 682         actionsInfos.add(new ActionPuntToController());
 683         if (snatPuntTimeout != 0) {
 684             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 685         }
 686         instructions.add(new InstructionApplyActions(actionsInfos));
 687
 688         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 689         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 690             5, flowRef, 0, 0,
 691             cookie, matches, instructions);
 692         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 693         return flowEntity;
 694     }
 695
 696     public void createOutboundTblEntry(Uint64 dpnId, Uint32 routerId, TypedWriteTransaction<Configuration> confTx) {
 697         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 698         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 699         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 700         mdsalManager.addFlow(confTx, tcpflowEntity);
 701
 702         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 703         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 704         mdsalManager.addFlow(confTx, udpflowEntity);
 705
 706         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 707         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 708         mdsalManager.addFlow(confTx, icmpDropFlow);
 709     }
 710
 711     @Nullable
 712     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 713         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 714         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 715         try {
 716             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 717                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 718                     .setSourceDpid(srcDpId)
 719                     .setDestinationDpid(dstDpId)
 720                     .setTunnelType(tunType)
 721                     .build());
 722             rpcResult = result.get();
 723             if (!rpcResult.isSuccessful()) {
 724                 tunType = TunnelTypeGre.class;
 725                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 726                     .setSourceDpid(srcDpId)
 727                     .setDestinationDpid(dstDpId)
 728                     .setTunnelType(tunType)
 729                     .build());
 730                 rpcResult = result.get();
 731                 if (!rpcResult.isSuccessful()) {
 732                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 733                             rpcResult.getErrors());
 734                 } else {
 735                     return rpcResult.getResult().getInterfaceName();
 736                 }
 737                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 738                         rpcResult.getErrors());
 739             } else {
 740                 return rpcResult.getResult().getInterfaceName();
 741             }
 742         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 743             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 744                     + "between {} and {}", srcDpId, dstDpId, e);
 745         }
 746
 747         return null;
 748     }
 749
 750     protected void installSnatMissEntryForPrimrySwch(Uint64 dpnId, String routerName, Uint32 routerId,
 751         TypedWriteTransaction<Configuration> confTx) {
 752
 753         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 754         // Install miss entry pointing to group
 755         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 756         mdsalManager.addFlow(confTx, flowEntity);
 757     }
 758
 759     protected void installSnatMissEntry(Uint64 dpnId, List<BucketInfo> bucketInfo,
 760             String routerName, Uint32 routerId) {
 761         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 762             dpnId, bucketInfo.get(0));
 763         // Install the select group
 764         Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
 765             NatUtil.getGroupIdKey(routerName));
 766         if (groupId == NatConstants.INVALID_ID) {
 767             LOG.error("installSnatMissEntry: Unable to obtain group ID for Key: {}", routerName);
 768             return;
 769         }
 770         GroupEntity groupEntity =
 771             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 772         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 773         mdsalManager.syncInstallGroup(groupEntity);
 774         // Install miss entry pointing to group
 775         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 776         if (flowEntity == null) {
 777             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 778                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 779                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 780             return;
 781         }
 782         mdsalManager.installFlow(flowEntity);
 783     }
 784
 785     void installGroup(Uint64 dpnId, String routerName, Uint32 groupId, List<BucketInfo> bucketInfo) {
 786         GroupEntity groupEntity =
 787             MDSALUtil.buildGroupEntity(dpnId, groupId.longValue(), routerName, GroupTypes.GroupAll, bucketInfo);
 788         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 789         mdsalManager.syncInstallGroup(groupEntity);
 790     }
 791
 792     private FlowEntity buildSnatFlowEntity(Uint64 dpId, String routerName, Uint32 routerId, Uint32 groupId) {
 793         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 794                 dpId, routerName, groupId);
 795         List<MatchInfo> matches = new ArrayList<>();
 796         matches.add(MatchEthernetType.IPV4);
 797         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 798                 MetaDataUtil.METADATA_MASK_VRFID));
 799
 800         List<ActionInfo> actionsInfo = new ArrayList<>();
 801         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 802                 routerId, routerName);
 803         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
 804         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 805         actionsInfo.add(new ActionGroup(groupId.longValue()));
 806         List<InstructionInfo> instructions = new ArrayList<>();
 807         instructions.add(new InstructionApplyActions(actionsInfo));
 808         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 809         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 810                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 811                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 812
 813         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 814         return flowEntity;
 815     }
 816
 817     private FlowEntity buildSnatFlowEntityForPrmrySwtch(Uint64 dpId, String routerName, Uint32 routerId) {
 818
 819         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 820             routerName);
 821         List<MatchInfo> matches = new ArrayList<>();
 822         matches.add(MatchEthernetType.IPV4);
 823         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 824                 MetaDataUtil.METADATA_MASK_VRFID));
 825
 826         List<InstructionInfo> instructions = new ArrayList<>();
 827         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 828
 829         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 830         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 831             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 832             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 833
 834         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 835         return flowEntity;
 836     }
 837
 838     // TODO : Replace this with ITM Rpc once its available with full functionality
 839     protected void installTerminatingServiceTblEntry(Uint64 dpnId, String routerName, Uint32 routerId,
 840         TypedWriteTransaction<Configuration> confTx) {
 841
 842         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 843             dpnId, routerName);
 844         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 845         if (flowEntity == null) {
 846             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 847                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 848                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 849                     dpnId, routerName);
 850             return;
 851         }
 852         mdsalManager.addFlow(confTx, flowEntity);
 853
 854     }
 855
 856     private FlowEntity buildTsFlowEntity(Uint64 dpId, String routerName, Uint32 routerId) {
 857         List<MatchInfo> matches = new ArrayList<>();
 858         matches.add(MatchEthernetType.IPV4);
 859         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 860                 idManager, routerId, routerName);
 861         matches.add(new MatchTunnelId(tunnelId));
 862         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
 863         List<InstructionInfo> instructions = new ArrayList<>();
 864         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId.longValue()),
 865                 MetaDataUtil.METADATA_MASK_VRFID));
 866         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 867         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 868                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
 869                 NwConstants.COOKIE_TS_TABLE, matches, instructions);
 870         return flowEntity;
 871     }
 872
 873     public String getFlowRefTs(Uint64 dpnId, short tableId, Uint32 routerID) {
 874         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 875                 .FLOWID_SEPARATOR + routerID;
 876     }
 877
 878     public static String getFlowRefSnat(Uint64 dpnId, short tableId, String routerID) {
 879         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 880                 .FLOWID_SEPARATOR + routerID;
 881     }
 882
 883     protected void handleSwitches(Uint64 dpnId, String routerName, Uint32 routerId, Uint64 primarySwitchId) {
 884         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 885         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 886         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 887         List<BucketInfo> listBucketInfo = new ArrayList<>();
 888
 889         if (ifNamePrimary != null) {
 890             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 891             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 892                     interfaceManager, ifNamePrimary, routerId, true);
 893             if (listActionInfoPrimary.isEmpty()) {
 894                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 895                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 896                         ifNamePrimary);
 897             }
 898         } else {
 899             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 900                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 901         }
 902         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 903
 904         listBucketInfo.add(0, bucketPrimary);
 905         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 906     }
 907
 908     List<BucketInfo> getBucketInfoForNonNaptSwitches(Uint64 nonNaptSwitchId,
 909                                                      Uint64 primarySwitchId, String routerName, Uint32 routerId) {
 910         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 911         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 912         List<BucketInfo> listBucketInfo = new ArrayList<>();
 913
 914         if (ifNamePrimary != null) {
 915             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 916                     ifNamePrimary);
 917             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 918                     interfaceManager, ifNamePrimary, routerId, true);
 919             if (listActionInfoPrimary.isEmpty()) {
 920                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 921                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 922                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 923             }
 924         } else {
 925             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 926                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 927         }
 928         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 929
 930         listBucketInfo.add(0, bucketPrimary);
 931         return listBucketInfo;
 932     }
 933
 934     protected void handlePrimaryNaptSwitch(Uint64 dpnId, String routerName, Uint32 routerId, Uuid externalNwUuid,
 935         TypedWriteTransaction<Configuration> confTx) {
 936
 937        /*
 938         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 939         */
 940
 941         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 942
 943 /*
 944         List<BucketInfo> listBucketInfo = new ArrayList<>();
 945         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 946         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 947         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 948         listBucketInfo.add(0, bucketPrimary);
 949 */
 950
 951         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 952         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 953         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 954         installNaptPfibEntry(dpnId, routerId, confTx);
 955         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 956         if (networkId != null) {
 957             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 958             if (vpnUuid != null) {
 959                 Uint32 extVpnId = NatUtil.getExternalVpnIdForExtNetwork(dataBroker, externalNwUuid);
 960                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 961                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 962                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 963                     if (extVpnId != null && extVpnId != NatConstants.INVALID_ID) {
 964                         installNaptPfibEntry(dpnId, extVpnId, null);
 965                     }
 966                     return Collections.emptyList();
 967                 });
 968             } else {
 969                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 970             }
 971         } else {
 972             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 973         }
 974     }
 975
 976     public void installNaptPfibEntry(Uint64 dpnId, Uint32 segmentId,
 977             @Nullable TypedWriteTransaction<Configuration> confTx) {
 978         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 979         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 980         if (confTx != null) {
 981             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 982         } else {
 983             mdsalManager.installFlow(naptPfibFlowEntity);
 984         }
 985     }
 986
 987     public FlowEntity buildNaptPfibFlowEntity(Uint64 dpId, Uint32 segmentId) {
 988
 989         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 990         List<MatchInfo> matches = new ArrayList<>();
 991         matches.add(MatchEthernetType.IPV4);
 992         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId.longValue()),
 993                 MetaDataUtil.METADATA_MASK_VRFID));
 994
 995         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 996         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 997         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
 998         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 999         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1000
1001         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1002         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1003             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1004             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1005         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1006         return flowEntity;
1007     }
1008
1009     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, Uint64 dpnId, Routers router,
1010                                          Uint32 routerId, String routerName, String externalIp) {
1011         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1012             dpnId, routerId, externalIp);
1013         Uuid networkId = router.getNetworkId();
1014         if (networkId == null) {
1015             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1016             return;
1017         }
1018         Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1019         // FLAT/VLAN case having external-subnet as VPN
1020         String externalSubnetVpn = null;
1021         if (externalSubnetList != null && !externalSubnetList.isEmpty()) {
1022             for (Uuid externalSubnetId : externalSubnetList) {
1023                 Optional<Subnets> externalSubnet = NatUtil
1024                     .getOptionalExternalSubnets(dataBroker, externalSubnetId);
1025                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1026                 if (externalSubnet.isPresent()) {
1027                     externalSubnetVpn = externalSubnetId.getValue();
1028                     advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1029                         externalSubnetVpn, routerId, routerName,
1030                         externalIp, networkId, router, confTx);
1031                 }
1032             }
1033             return;
1034         }
1035         // VXVLAN/GRE case having Internet-VPN
1036         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1037         if (vpnName == null) {
1038             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1039                     + "configuration {} in router {}", networkId, externalIp, routerId);
1040             return;
1041         }
1042         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1043             externalIp, networkId, router, confTx);
1044         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1045             dpnId, routerId, externalIp);
1046     }
1047
1048     public void advToBgpAndInstallFibAndTsFlows(final Uint64 dpnId, final short tableId, final String vpnName,
1049                                                 final Uint32 routerId, final String routerName, final String externalIp,
1050                                                 final Uuid extNetworkId, @Nullable final Routers router,
1051                                                 final TypedWriteTransaction<Configuration> confTx) {
1052         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1053                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1054         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1055         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1056         if (rd == null || rd.isEmpty()) {
1057             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1058             return;
1059         }
1060         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1061         if (extNwProvType == null) {
1062             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1063             return;
1064         }
1065         if (extNwProvType == ProviderTypes.VXLAN) {
1066             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1067                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1068             return;
1069         }
1070         //Generate VPN label for the external IP
1071         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1072             .setIpPrefix(externalIp).build();
1073         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1074
1075         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1076         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1077             if (result.isSuccessful()) {
1078                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1079                 GenerateVpnLabelOutput output = result.getResult();
1080                 final Uint32 label = output.getLabel();
1081
1082                 int externalIpInDsFlag = 0;
1083                 //Get IPMaps from the DB for the router ID
1084                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1085                 for (IpMap dbIpMap : dbIpMaps) {
1086                     String dbExternalIp = dbIpMap.getExternalIp();
1087                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1088                     if (dbExternalIp.contains(externalIp)) {
1089                         String dbInternalIp = dbIpMap.getInternalIp();
1090                         IpMapKey dbIpMapKey = dbIpMap.key();
1091                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1092                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1093                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1094                             .setExternalIp(dbExternalIp).setLabel(label).build();
1095                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1096                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1097                         externalIpInDsFlag++;
1098                     }
1099                 }
1100                 if (externalIpInDsFlag <= 0) {
1101                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1102                             + "Failed to update label {} for routerId {} in DS",
1103                             externalIp, label, routerId);
1104                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1105                         + " found in DS for router %s", label, externalIp, routerId);
1106                     return Futures.immediateFailedFuture(new Exception(errMsg));
1107                 }
1108                 //Inform BGP
1109                 Uint32 l3vni = Uint32.ZERO;
1110                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1111                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
1112                 }
1113                 Routers extRouter = router != null ? router :
1114                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1115                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1116                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1117                     RouteOrigin.STATIC, dpnId);
1118
1119                 //Install custom FIB routes
1120                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1121                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1122                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1123                         extNwProvType);
1124                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1125
1126                 //Install custom FIB routes - FIB table.
1127                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1128                         routerName, externalIp);
1129                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1130                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1131                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1132                 }
1133                 String externalVpn = vpnName;
1134                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1135                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1136                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1137                             externalSubnetId);
1138                     if (externalSubnet.isPresent()) {
1139                         externalVpn =  externalSubnetId.getValue();
1140                     }
1141                 }
1142                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1143                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1144                     .setVpnName(externalVpn)
1145                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1146                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1147                     .setInstruction(fibTableCustomInstructions).build();
1148                 return fibService.createFibEntry(input);
1149             } else {
1150                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1151                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1152                     externalIp, vpnName, result.getErrors());
1153                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1154             }
1155         }, MoreExecutors.directExecutor());
1156
1157         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1158
1159             @Override
1160             public void onFailure(@NonNull Throwable error) {
1161                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1162             }
1163
1164             @Override
1165             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1166                 if (result.isSuccessful()) {
1167                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1168                             externalIp);
1169                 } else {
1170                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1171                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1172                 }
1173             }
1174         }, MoreExecutors.directExecutor());
1175     }
1176
1177     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1178             String externalIp) {
1179         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1180         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1181         Uint32 externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1182                 externalIp, router);
1183         int instructionIndex = 0;
1184         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1185             Uint64 subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId.longValue());
1186             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1187                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1188             instructionIndex++;
1189         }
1190
1191         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1192         return fibTableCustomInstructions;
1193     }
1194
1195     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, short tableId,
1196         TypedWriteTransaction<Configuration> confTx) {
1197         List<MatchInfo> matches = new ArrayList<>();
1198         matches.add(MatchEthernetType.MPLS_UNICAST);
1199         matches.add(new MatchMplsLabel(serviceId.longValue()));
1200
1201         List<Instruction> instructions = new ArrayList<>();
1202         List<ActionInfo> actionsInfos = new ArrayList<>();
1203         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1204         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1205         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1206         instructions.add(writeInstruction);
1207         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1208
1209         // Install the flow entry in L3_LFIB_TABLE
1210         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1211
1212         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1213             10, flowRef, 0, 0,
1214             COOKIE_VM_LFIB_TABLE, matches, instructions);
1215
1216         mdsalManager.addFlow(confTx, dpId, flowEntity);
1217
1218         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1219     }
1220
1221     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId, Uint32 l3Vni,
1222         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1223         ProviderTypes extNwProvType) {
1224
1225         List<MatchInfo> mkMatches = new ArrayList<>();
1226
1227         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1228             dpnId, serviceId, customInstructions);
1229
1230         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1231             mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
1232         } else {
1233             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
1234         }
1235
1236         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1237             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
1238                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY,
1239                 String.format("%s:%s", "TST Flow Entry ", serviceId), 0, 0,
1240                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(serviceId.longValue()))),
1241                 mkMatches, customInstructions);
1242
1243         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1244     }
1245
1246     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(Uint32 routerId) {
1247         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1248             new RouterIdsKey(routerId)).build();
1249     }
1250
1251     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
1252         return NatConstants.SNAT_FLOWID_PREFIX + dpnId.toString() + NwConstants.FLOWID_SEPARATOR + tableId
1253                 + NwConstants.FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1254     }
1255
1256     @Override
1257     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1258         LOG.trace("update : origRouter: {} updatedRouter: {}", original, update);
1259         String routerName = original.getRouterName();
1260         Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1261         if (routerId == NatConstants.INVALID_ID) {
1262             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1263             return;
1264         }
1265         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1266             List<ListenableFuture<Void>> futures = new ArrayList<>();
1267             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1268                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1269                     Uint32 bgpVpnId = NatConstants.INVALID_ID;
1270                     Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1271                     if (bgpVpnUuid != null) {
1272                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1273                     }
1274                     //BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1275                     /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1276                      * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1277                      */
1278                     Uint64 dpnId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1279                     boolean isPrimaryNaptSwitchNotSelected = (dpnId == null || dpnId.equals(Uint64
1280                             .valueOf(BigInteger.ZERO)));
1281                     Uuid networkId = original.getNetworkId();
1282                     // Check if its update on SNAT flag
1283                     boolean originalSNATEnabled = original.isEnableSnat();
1284                     boolean updatedSNATEnabled = update.isEnableSnat();
1285                     LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1286                             + "as {} and {} with Elected Dpn {}(isPrimaryNaptSwitchNotSelected:{})",
1287                         originalSNATEnabled, updatedSNATEnabled, dpnId, isPrimaryNaptSwitchNotSelected);
1288                     // Cluster Reboot Case Handling
1289                     // 1. DPN not elected during add event(due to none of the OVS connected)
1290                     // 2. Update event called with changes of parameters(but enableSnat is not changed)
1291                     // 3. First Elect dpnId and process other changes with valid dpnId
1292                     if (originalSNATEnabled != updatedSNATEnabled || isPrimaryNaptSwitchNotSelected) {
1293                         if (originalSNATEnabled && !updatedSNATEnabled) {
1294                             if (isPrimaryNaptSwitchNotSelected) {
1295                                 LOG.info("No Action to be taken when SNAT is disabled "
1296                                     + "with no Napt Switch Election for Router {}", routerName);
1297                                 return;
1298                             }
1299                             //SNAT disabled for the router
1300                             Uuid networkUuid = original.getNetworkId();
1301                             LOG.info("update : SNAT disabled for Router {}", routerName);
1302                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1303                             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1304                             handleDisableSnat(original, networkUuid, externalIps, false, vpnName,
1305                                     dpnId, routerId, removeFlowInvTx);
1306                         }  else if (updatedSNATEnabled) {
1307                             LOG.info("update : SNAT enabled for Router {}", routerName);
1308                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid,
1309                                 true, writeFlowInvTx);
1310                             if (isPrimaryNaptSwitchNotSelected) {
1311                                 dpnId = selectNewNAPTSwitch(routerName);
1312                                 if (dpnId != null && !dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1313                                     handleEnableSnat(update, routerId, dpnId, bgpVpnId, removeFlowInvTx);
1314                                 } else {
1315                                     LOG.error("update : Failed to elect Napt Switch During update event"
1316                                         + " of router {}", routerName);
1317                                 }
1318                             }
1319                         }
1320                         LOG.info("update : no need to process external/subnet changes as it's will taken care"
1321                             + "in handleDisableSnat/handleEnableSnat");
1322                         return;
1323                     }
1324                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1325                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1326                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1327                     }
1328
1329                     if (updatedSNATEnabled != originalSNATEnabled) {
1330                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1331                                 + "handleDisableSnat/handleEnableSnat");
1332                         return;
1333                     }
1334                     //Check if the Update is on External IPs
1335                     LOG.debug("update : Checking if this is update on External IPs for router {}", routerName);
1336                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1337                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1338
1339                     //Check if the External IPs are removed during the update.
1340                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1341                     removedExternalIps.removeAll(updatedExternalIps);
1342                     if (removedExternalIps.size() > 0) {
1343                         LOG.debug("update : Start processing of the External IPs removal for router {}", routerName);
1344                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1345                                 removedExternalIps, original.getExtGwMacAddress(),
1346                                 dpnId, networkId);
1347
1348                         for (String removedExternalIp : removedExternalIps) {
1349                 /*
1350                     1) Remove the mappings in the IntExt IP model which has external IP.
1351                     2) Remove the external IP in the ExternalCounter model.
1352                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1353                        least loaded external IP.
1354                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1355                     4) Increase the count of the allocated external IP by one.
1356                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1357                      i.e. the route for the external IP is absent.
1358                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1359                      the removed external IPs and also from the model.
1360                     7) Advertise to the BGP for removing the route for the removed external IPs.
1361                  */
1362
1363                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1364                             String externalIp = externalIpParts[0];
1365                             String externalIpPrefix = externalIpParts[1];
1366                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1367
1368                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1369                                     + "entries for removed external IP {}", externalIpAddrStr);
1370                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1371                             String vpnName = "";
1372                             if (vpnUuId != null) {
1373                                 vpnName = vpnUuId.getValue();
1374                             }
1375                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1376                                     update.getExtGwMacAddress(), removeFlowInvTx);
1377
1378                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1379                             //Get the internal IPs which are associated to the removed external IPs
1380                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1381                             List<String> removedInternalIps = new ArrayList<>();
1382                             for (IpMap ipMap : ipMaps) {
1383                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1384                                     removedInternalIps.add(ipMap.getInternalIp());
1385                                 }
1386                             }
1387
1388                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1389                             for (String removedInternalIp : removedInternalIps) {
1390                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1391                                                 + "router ID {} from the IntExtIP model",
1392                                         removedInternalIp, routerId);
1393                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1394                             }
1395
1396                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1397                                             + "router ID {} from the ExternalIpsCounter model.",
1398                                     externalIpAddrStr, routerId);
1399                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1400
1401                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1402                                     + "whose external IPs were removed.");
1403                             for (String removedInternalIp : removedInternalIps) {
1404                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1405                                         removedInternalIp, writeFlowInvTx);
1406                             }
1407                             LOG.debug("update : Remove the NAPT translation entries from "
1408                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1409                             //Get the internalIP and internal Port which were associated to the removed external IP.
1410                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1411                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1412                                     .builder(IntextIpPortMap.class)
1413                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1414                             Optional<IpPortMapping> ipPortMapping;
1415                             try {
1416                                 ipPortMapping = SingleTransactionDataBroker
1417                                             .syncReadOptional(dataBroker,
1418                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1419                             } catch (ReadFailedException e) {
1420                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1421                                 ipPortMapping = Optional.absent();
1422                             }
1423
1424                             if (ipPortMapping.isPresent()) {
1425                                 for (IntextIpProtocolType intextIpProtocolType :
1426                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1427                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1428                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1429                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1430                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1431                                             List<String> removedInternalIpPorts =
1432                                                     protoTypesIntIpPortsMap.get(protoType);
1433                                             if (removedInternalIpPorts != null) {
1434                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1435                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1436                                             } else {
1437                                                 removedInternalIpPorts = new ArrayList<>();
1438                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1439                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1440                                             }
1441                                         }
1442                                     }
1443                                 }
1444                             }
1445
1446                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1447                             // the removed external IP.
1448                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1449                                     protoTypesIntIpPortsMap.entrySet();
1450                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1451                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1452                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1453                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1454                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1455                                     // Remove the IP port map from the intext-ip-port-map model,
1456                                     // which were containing the removed external IP
1457                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1458                                             protocolType);
1459                                     //Remove the IP port incomint packer map.
1460                                     naptPacketInHandler.removeIncomingPacketMap(
1461                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1462                                     String[] removedInternalIpPortParts = removedInternalIpPort
1463                                             .split(NatConstants.COLON_SEPARATOR);
1464                                     if (removedInternalIpPortParts.length == 2) {
1465                                         String removedInternalIp = removedInternalIpPortParts[0];
1466                                         String removedInternalPort = removedInternalIpPortParts[1];
1467                                         List<String> removedInternalPortsList =
1468                                                 internalIpPortMap.get(removedInternalPort);
1469                                         if (removedInternalPortsList != null) {
1470                                             removedInternalPortsList.add(removedInternalPort);
1471                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1472                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1473                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1474                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1475                                             //Remove the NAPT translation entries from Outbound NAPT table
1476                                             naptEventHandler.removeNatFlows(dpnId,
1477                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1478                                                 routerId, removedInternalIp,
1479                                                 Integer.parseInt(removedInternalPort),
1480                                                 protocolType.getName());
1481                                             naptEventHandler.removeNatFlows(dpnId,
1482                                                 NwConstants.INBOUND_NAPT_TABLE,
1483                                                 routerId, removedInternalIp,
1484                                                 Integer.parseInt(removedInternalPort),
1485                                                 protocolType.getName());
1486                                         } else {
1487                                             removedInternalPortsList = new ArrayList<>();
1488                                             removedInternalPortsList.add(removedInternalPort);
1489                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1490                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1491                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1492                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1493                                             //Remove the NAPT translation entries from Outbound NAPT table
1494                                             naptEventHandler.removeNatFlows(dpnId,
1495                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1496                                                 routerId, removedInternalIp,
1497                                                 Integer.parseInt(removedInternalPort),
1498                                                 protocolType.getName());
1499                                             naptEventHandler.removeNatFlows(dpnId,
1500                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1501                                                 Integer.parseInt(removedInternalPort),
1502                                                 protocolType.getName());
1503                                         }
1504                                     }
1505                                 }
1506                             }
1507
1508                             // Delete the entry from SnatIntIpPortMap DS
1509                             Set<String> internalIps = internalIpPortMap.keySet();
1510                             for (String internalIp : internalIps) {
1511                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1512                                         + "model SnatIntIpPortMap", internalIp);
1513                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1514                             }
1515
1516                             naptManager.removeNaptPortPool(externalIp);
1517                         }
1518                         LOG.debug(
1519                                 "update : End processing of the External IPs removal for router {}", routerName);
1520                     }
1521
1522                     //Check if the External IPs are added during the update.
1523                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1524                     addedExternalIps.removeAll(originalExternalIps);
1525                     if (addedExternalIps.size() != 0) {
1526                         LOG.debug("update : Start processing of the External IPs addition for router {}",
1527                             routerName);
1528                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1529                                 update.getExtGwMacAddress(), dpnId,
1530                                 update.getNetworkId());
1531
1532                         for (String addedExternalIp : addedExternalIps) {
1533                 /*
1534                     1) Do nothing in the IntExtIp model.
1535                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1536                  */
1537                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1538                             String externalIp = externalIpParts[0];
1539                             String externalIpPrefix = externalIpParts[1];
1540                             String externalpStr = externalIp + "/" + externalIpPrefix;
1541                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1542                                             + "router ID {} in the ExternalIpsCounter model.",
1543                                     externalpStr, routerId);
1544                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1545                             subnetRegisterMapping(update, routerId);
1546                             LOG.info("update : Installing fib flow fo newly added Ips");
1547                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1548                         }
1549                         LOG.debug(
1550                                 "update : End processing of the External IPs addition during the update operation");
1551                     }
1552
1553                     //Check if its Update on subnets
1554                     LOG.debug("update : Checking if this is update on subnets for router {}", routerName);
1555                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1556                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1557                     Set<Uuid> addedSubnetIds =
1558                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1559                     if (originalSubnetIds != null) {
1560                         addedSubnetIds.removeAll(originalSubnetIds);
1561                     }
1562
1563                     //Check if the Subnet IDs are added during the update.
1564                     if (addedSubnetIds.size() != 0) {
1565                         LOG.debug(
1566                                 "update : Start processing of the Subnet IDs addition for router {}", routerName);
1567                         for (Uuid addedSubnetId : addedSubnetIds) {
1568                 /*
1569                     1) Select the least loaded external IP for the subnet and store the mapping of the
1570                     subnet IP and the external IP in the IntExtIp model.
1571                     2) Increase the count of the selected external IP by one.
1572                     3) Advertise to the BGP if external IP is allocated for the first time for the
1573                     router i.e. the route for the external IP is absent.
1574                  */
1575                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1576                             if (subnetIp != null) {
1577                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1578                                         writeFlowInvTx);
1579                             }
1580                         }
1581                         LOG.debug("update : End processing of the Subnet IDs addition for router {}", routerName);
1582                     }
1583
1584                     //Check if the Subnet IDs are removed during the update.
1585                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1586                     removedSubnetIds.removeAll(updatedSubnetIds);
1587                     if (removedSubnetIds.size() != 0) {
1588                         LOG.debug(
1589                                 "update : Start processing of the Subnet IDs removal for router {}", routerName);
1590                         for (Uuid removedSubnetId : removedSubnetIds) {
1591                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1592                             if (subnetAddr != null) {
1593                     /*
1594                         1) Remove the subnet IP and the external IP in the IntExtIp map
1595                         2) Decrease the count of the coresponding external IP by one.
1596                         3) Advertise to the BGP for removing the routes of the corresponding external
1597                         IP if its not allocated to any other internal IP.
1598                     */
1599
1600                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1601                                         subnetAddr[0] + "/" + subnetAddr[1]);
1602                                 if (externalIp == null) {
1603                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1604                                             routerId, subnetAddr[0]);
1605                                     return;
1606                                 }
1607
1608                                 naptManager.updateCounter(routerId, externalIp, false);
1609                                 // Traverse entire model of external-ip counter whether external ip is not
1610                                 // used by any other internal ip in any router
1611                                 if (!isExternalIpAllocated(externalIp)) {
1612                                     LOG.debug("update : external ip is not allocated to any other "
1613                                             + "internal IP so proceeding to remove routes");
1614                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1615                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1616                                             removeFlowInvTx);
1617                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1618                                                     + "router {} with networkId {} and externalIp {}",
1619                                             dpnId, routerId, networkId, externalIp);
1620                                 }
1621
1622                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1623                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1624                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1625                             }
1626                         }
1627                         LOG.debug("update : End processing of the Subnet IDs removal for router {}", routerName);
1628                     }
1629                 }));
1630             }));
1631             return futures;
1632         }, NatConstants.NAT_DJC_MAX_RETRIES);
1633     }
1634
1635     private boolean isExternalIpAllocated(String externalIp) {
1636         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1637         Optional<ExternalIpsCounter> externalCountersData;
1638         try {
1639             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1640                         LogicalDatastoreType.OPERATIONAL, id);
1641         } catch (ReadFailedException e) {
1642             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1643             externalCountersData = Optional.absent();
1644         }
1645         if (externalCountersData.isPresent()) {
1646             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1647             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1648                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1649                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1650                         if (externalIpCount.getCounter().toJava() != 0) {
1651                             return true;
1652                         }
1653                         break;
1654                     }
1655                 }
1656             }
1657         }
1658         return false;
1659     }
1660
1661     private void allocateExternalIp(Uint64 dpnId, Routers router, Uint32 routerId, String routerName,
1662             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1663         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1664         try {
1665             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1666             if (address instanceof Inet6Address) {
1667                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1668                 return;
1669             }
1670         } catch (UnknownHostException e) {
1671             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1672             return;
1673         }
1674         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1675         if (leastLoadedExtIpAddr != null) {
1676             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1677             String leastLoadedExtIp = externalIpParts[0];
1678             String leastLoadedExtIpPrefix = externalIpParts[1];
1679             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1680             subnetIp = subnetIpParts[0];
1681             String subnetIpPrefix = subnetIpParts[1];
1682             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1683             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1684                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1685                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1686             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1687
1688
1689             // Check if external IP is already assigned a route. (i.e. External IP is previously
1690             // allocated to any of the subnets)
1691             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1692             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1693             Uint32 label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1694             if (label != null) {
1695                 //update
1696                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1697                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1698                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1699                     label, internalIp, leastLoadedExtIpAddrStr);
1700                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1701                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1702                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1703                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1704                 return;
1705             }
1706
1707             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1708             // for the first time and hence not having a route.
1709             //Get the VPN Name using the network ID
1710             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1711             if (vpnName != null) {
1712                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1713                 if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1714                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1715                         + "added after gateway-set");
1716                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1717                     if (dpnId == null || dpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1718                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1719                         return;
1720                     }
1721                 }
1722                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1723                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1724                     writeFlowInvTx);
1725             }
1726         }
1727     }
1728
1729     @Nullable
1730     protected Uint32 checkExternalIpLabel(Uint32 routerId, String externalIp) {
1731         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1732         for (IpMap ipMap : ipMaps) {
1733             if (ipMap.getExternalIp().equals(externalIp)) {
1734                 if (ipMap.getLabel() != null) {
1735                     return ipMap.getLabel();
1736                 }
1737             }
1738         }
1739         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1740         return null;
1741     }
1742
1743     @Override
1744     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1745         LOG.trace("remove : Router delete method");
1746         /*
1747         ROUTER DELETE SCENARIO
1748         1) Get the router ID from the event.
1749         2) Build the cookie information from the router ID.
1750         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1751         4) Build the flow with the cookie value.
1752         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1753         6) Remove the flows from the other switches which points to the primary and secondary
1754          switches for the flows related the router ID.
1755         7) Get the list of external IP address maintained for the router ID.
1756         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1757         9) Withdraw the corresponding routes from the BGP.
1758          */
1759
1760         if (identifier == null || router == null) {
1761             LOG.error("remove : returning without processing since routers is null");
1762             return;
1763         }
1764
1765         String routerName = router.getRouterName();
1766         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1767             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1768                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1769                         routerName);
1770                 Uint32 routerId = NatUtil.getVpnId(dataBroker, routerName);
1771                 if (routerId == NatConstants.INVALID_ID) {
1772                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1773                             routerName);
1774                     return;
1775                 }
1776                 Uint32 bgpVpnId = NatConstants.INVALID_ID;
1777                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1778                 if (bgpVpnUuid != null) {
1779                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1780                 }
1781                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1782                         tx);
1783                 Uuid networkUuid = router.getNetworkId();
1784
1785                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1786                 if (primarySwitchId == null || primarySwitchId.equals(Uint64.ZERO)) {
1787                     // No NAPT switch for external router, probably because the router is not attached to
1788                     // any
1789                     // internal networks
1790                     LOG.debug(
1791                             "No NAPT switch for router {}, check if router is attached to any internal "
1792                                     + "network",
1793                             routerName);
1794                     return;
1795                 } else {
1796                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1797                     final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
1798                     handleDisableSnat(router, networkUuid, externalIps, true, vpnName, primarySwitchId,
1799                             routerId, tx);
1800                 }
1801                 if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, routerName)
1802                     == NatConstants.INVALID_ID) {
1803                     LOG.error("remove: Unable to release VNI for router - {}", routerName);
1804                 }
1805             })), NatConstants.NAT_DJC_MAX_RETRIES);
1806     }
1807
1808     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1809                                   boolean routerFlag, @Nullable String vpnName, Uint64 naptSwitchDpnId,
1810                                   Uint32 routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1811         LOG.info("handleDisableSnat : Entry");
1812         String routerName = router.getRouterName();
1813         try {
1814             if (routerFlag) {
1815                 removeNaptSwitch(routerName);
1816             } else {
1817                 updateNaptSwitch(routerName, Uint64.valueOf(BigInteger.ZERO));
1818             }
1819
1820             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1821             naptManager.removeExternalCounter(routerId);
1822
1823             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1824             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(Uint64.valueOf(BigInteger.ZERO))) {
1825                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1826                     + "router ID {} from RouterNaptSwitch model", routerId);
1827                 return;
1828             }
1829             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1830                     networkUuid);
1831             if (extNwProvType == null) {
1832                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1833                 return;
1834             }
1835             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1836             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1837                     externalSubnetList, removeFlowInvTx, extNwProvType);
1838             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1839             String externalSubnetVpn = null;
1840             for (Uuid externalSubnetId : externalSubnetList) {
1841                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1842                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1843                 if (externalSubnet.isPresent()) {
1844                     externalSubnetVpn =  externalSubnetId.getValue();
1845                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1846                             router.getExtGwMacAddress(), removeFlowInvTx);
1847                 }
1848             }
1849             if (externalSubnetVpn == null) {
1850                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1851                         router.getExtGwMacAddress(), removeFlowInvTx);
1852             }
1853             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1854             // for the router ID.
1855             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1856                     + "router ID {} in the DS", routerId);
1857             naptManager.removeMapping(routerId);
1858         } catch (InterruptedException | ExecutionException e) {
1859             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1860         }
1861         LOG.info("handleDisableSnat : Exit");
1862     }
1863
1864     // TODO Clean up the exception handling
1865     @SuppressWarnings("checkstyle:IllegalCatch")
1866     public void handleDisableSnatInternetVpn(String routerName, Uint32 routerId, Uuid networkUuid,
1867                                              @NonNull Collection<String> externalIps,
1868                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1869         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1870                 + "with internet vpn {}", routerName, vpnId);
1871         try {
1872             Uint64 naptSwitchDpnId = null;
1873             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1874                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1875             Optional<RouterToNaptSwitch> rtrToNapt;
1876             try {
1877                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1878                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1879             } catch (ReadFailedException e) {
1880                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1881                 rtrToNapt = Optional.absent();
1882             }
1883             if (rtrToNapt.isPresent()) {
1884                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1885             }
1886             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1887
1888             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1889                     writeFlowInvTx);
1890             try {
1891                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1892                 if (extGwMacAddress != null) {
1893                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1894                             + "External Router ID {}", extGwMacAddress, routerId);
1895                 } else {
1896                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1897                             + "External Router ID {}", routerId);
1898                     return;
1899                 }
1900                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1901                         writeFlowInvTx);
1902             } catch (Exception ex) {
1903                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1904                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1905             }
1906             if (NatUtil.releaseId(idManager, NatConstants.ODL_VNI_POOL_NAME, vpnId) == NatConstants.INVALID_ID) {
1907                 LOG.error("handleDisableSnatInternetVpn : Unable to release VNI for vpnId {} ", vpnId);
1908             }
1909         } catch (InterruptedException | ExecutionException e) {
1910             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1911                     + "with internet vpn {}", routerName, vpnId, e);
1912         }
1913         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1914                 routerName, vpnId);
1915     }
1916
1917     // TODO Clean up the exception handling
1918     @SuppressWarnings("checkstyle:IllegalCatch")
1919     public void updateNaptSwitch(String routerName, Uint64 naptSwitchId) {
1920         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1921             .setPrimarySwitchId(naptSwitchId).build();
1922         try {
1923             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1924                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1925         } catch (Exception ex) {
1926             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1927                 naptSwitchId, routerName);
1928         }
1929         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1930             naptSwitchId, routerName);
1931     }
1932
1933     protected void removeNaptSwitch(String routerName) {
1934         // Remove router and switch from model
1935         InstanceIdentifier<RouterToNaptSwitch> id =
1936             InstanceIdentifier.builder(NaptSwitches.class)
1937                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1938         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1939         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1940         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1941         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1942     }
1943
1944     public void removeNaptFlowsFromActiveSwitch(Uint32 routerId, String routerName,
1945                                                 Uint64 dpnId, Uuid networkId, String vpnName,
1946                                                 @NonNull Collection<String> externalIps,
1947                                                 Collection<Uuid> externalSubnetList,
1948                                                 TypedReadWriteTransaction<Configuration> confTx,
1949                                                 ProviderTypes extNwProvType)
1950             throws InterruptedException, ExecutionException {
1951
1952         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1953         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1954
1955         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1956         // traffic which comes from the  VMs of the NAPT switches)
1957         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1958         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1959
1960         LOG.info(
1961             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1962                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1963         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1964
1965         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1966         // traffic which comes from the VMs of the non NAPT switches)
1967         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1968                 elanManager, idManager, routerId, routerName);
1969         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, Uint32.valueOf(tunnelId.longValue()));
1970         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1971         LOG.info(
1972             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1973                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1974         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1975
1976         //Remove the flow table 25->44 from NAPT Switch
1977         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1978             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1979         }
1980
1981         //Remove the Outbound flow entry which forwards the packet to FIB Table
1982         LOG.info(
1983             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1984                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1985
1986         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1987             NwConstants.IP_PROT_TCP);
1988         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1989             outboundTcpNatFlowRef);
1990         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1991
1992         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1993             NwConstants.IP_PROT_UDP);
1994         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1995             outboundUdpNatFlowRef);
1996         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1997
1998         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1999             NwConstants.IP_PROT_ICMP);
2000         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2001             icmpDropFlowRef);
2002         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
2003         boolean lastRouterOnExternalNetwork =
2004             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
2005         if (lastRouterOnExternalNetwork) {
2006             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
2007         }
2008         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
2009         // External Subnet Vpn Id.
2010         for (Uuid externalSubnetId : externalSubnetList) {
2011             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2012             if (subnetVpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
2013                 dataBroker, externalSubnetId, routerName, dpnId)) {
2014                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
2015                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
2016                     natPfibSubnetFlowRef);
2017                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2018                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
2019                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
2020                     subnetVpnId, dpnId);
2021             }
2022         }
2023
2024         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
2025         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
2026         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
2027
2028         LOG.info(
2029             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
2030             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
2031         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
2032
2033         if (lastRouterOnExternalNetwork) {
2034             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
2035             // - This does not work since ext-routers is deleted already - no network info
2036             //Get the VPN ID from the ExternalNetworks model
2037             Uint32 vpnId = NatConstants.INVALID_ID;
2038             if (vpnName == null || vpnName.isEmpty()) {
2039                 // ie called from router delete cases
2040                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2041                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
2042                 if (vpnUuid != null) {
2043                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2044                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2045                               + "or disableSNAT scenario", vpnId, networkId);
2046                 }
2047             } else {
2048                 // ie called from disassociate vpn case
2049                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2050                     vpnName);
2051                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2052                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2053                     vpnId);
2054             }
2055
2056             if (vpnId != NatConstants.INVALID_ID) {
2057                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2058                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2059                 FlowEntity natPfibVpnFlowEntity =
2060                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2061                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2062                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2063                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2064             }
2065         }
2066
2067         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2068         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2069         if (ipPortMapping == null) {
2070             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2071             return;
2072         }
2073
2074         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2075             String protocol = intextIpProtocolType.getProtocol().name();
2076             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2077                 String ipPortInternal = ipPortMap.getIpPortInternal();
2078                 String[] ipPortParts = ipPortInternal.split(":");
2079                 if (ipPortParts.length != 2) {
2080                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2081                     return;
2082                 }
2083                 String internalIp = ipPortParts[0];
2084                 String internalPort = ipPortParts[1];
2085
2086                 //Build the flow for the outbound NAPT table
2087                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2088                     + NatConstants.COLON_SEPARATOR + internalPort);
2089                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2090                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2091                 FlowEntity outboundNaptFlowEntity =
2092                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2093
2094                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2095                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2096                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2097
2098                  //Build the flow for the inbound NAPT table
2099                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2100                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2101                 FlowEntity inboundNaptFlowEntity =
2102                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2103
2104                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2105                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2106                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2107             }
2108         }
2109     }
2110
2111     protected void removeNaptFibExternalOutputFlows(Uint32 routerId, Uint64 dpnId, Uuid networkId,
2112                                                     @NonNull Collection<String> externalIps,
2113                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2114             throws ExecutionException, InterruptedException {
2115         Uint32 extVpnId = NatConstants.INVALID_ID;
2116         if (networkId != null) {
2117             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2118             if (vpnUuid != null) {
2119                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2120             } else {
2121                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2122             }
2123         } else {
2124             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2125             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2126         }
2127         if (extVpnId == NatConstants.INVALID_ID) {
2128             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2129             extVpnId = routerId;
2130         }
2131         for (String ip : externalIps) {
2132             String extIp = removeMaskFromIp(ip);
2133             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2134             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2135                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2136                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2137             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2138             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2139         }
2140     }
2141
2142     private String removeMaskFromIp(String ip) {
2143         if (ip != null && !ip.trim().isEmpty()) {
2144             return ip.split("/")[0];
2145         }
2146         return ip;
2147     }
2148
2149     public void removeNaptFlowsFromActiveSwitchInternetVpn(Uint32 routerId, String routerName,
2150                                                            Uint64 dpnId, Uuid networkId, String vpnName,
2151                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2152             throws ExecutionException, InterruptedException {
2153         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2154         Uint64 cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2155
2156         //Remove the NAPT PFIB TABLE entry
2157         Uint32 vpnId = NatConstants.INVALID_ID;
2158         if (vpnName != null) {
2159             // ie called from disassociate vpn case
2160             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2161                     + "with vpnName {}", vpnName);
2162             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2163             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2164                     vpnId);
2165         }
2166
2167         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2168                 networkId, routerName, dpnId)) {
2169             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2170             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2171             FlowEntity natPfibVpnFlowEntity =
2172                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2173             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2174                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2175             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2176
2177             // Remove IP-PORT active NAPT entries and release port from IdManager
2178             // For the router ID get the internal IP , internal port and the corresponding
2179             // external IP and external Port.
2180             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2181             if (ipPortMapping == null) {
2182                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2183                 return;
2184             }
2185             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2186                 String protocol = intextIpProtocolType.getProtocol().name();
2187                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2188                     String ipPortInternal = ipPortMap.getIpPortInternal();
2189                     String[] ipPortParts = ipPortInternal.split(":");
2190                     if (ipPortParts.length != 2) {
2191                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2192                                 + "and port");
2193                         return;
2194                     }
2195                     String internalIp = ipPortParts[0];
2196                     String internalPort = ipPortParts[1];
2197
2198                     //Build the flow for the outbound NAPT table
2199                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2200                             + NatConstants.COLON_SEPARATOR + internalPort);
2201                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2202                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2203                     FlowEntity outboundNaptFlowEntity =
2204                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2205
2206                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2207                             + "active switch with the DPN ID {} and router ID {}",
2208                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2209                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2210
2211                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2212                     final String externalIp = ipPortExternal.getIpAddress();
2213
2214                     //Build the flow for the inbound NAPT table
2215                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2216                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2217                     FlowEntity inboundNaptFlowEntity =
2218                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2219
2220                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2221                             + "active active switch with the DPN ID {} and router ID {}",
2222                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2223                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2224
2225                     // Finally release port from idmanager
2226                     String internalIpPort = internalIp + ":" + internalPort;
2227                     naptManager.removePortFromPool(internalIpPort, externalIp);
2228
2229                     //Remove sessions from models
2230                     naptManager.removeIpPortMappingForRouterID(routerId);
2231                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2232                 }
2233             }
2234         } else {
2235             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2236         }
2237     }
2238
2239     public void removeFlowsFromNonActiveSwitches(Uint32 routerId, String routerName,
2240             Uint64 naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2241             throws ExecutionException, InterruptedException {
2242         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2243
2244         // Remove the flows from the other switches which points to the primary and secondary switches
2245         // for the flows related the router ID.
2246         List<Uint64> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2247         if (allSwitchList.isEmpty()) {
2248             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2249             return;
2250         }
2251         for (Uint64 dpnId : allSwitchList) {
2252             if (!naptSwitchDpnId.equals(dpnId)) {
2253                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2254
2255                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2256                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2257                 FlowEntity preSnatFlowEntity =
2258                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2259
2260                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2261                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2262                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2263
2264                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2265                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2266                     NatUtil.getGroupIdKey(routerName));
2267                 if (groupId != NatConstants.INVALID_ID) {
2268                     LOG.info(
2269                         "removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2270                             + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2271                     mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId.longValue());
2272                 } else {
2273                     LOG.error("removeFlowsFromNonActiveSwitches: Unable to obtained groupID for router:{}", routerName);
2274                 }
2275             }
2276         }
2277     }
2278
2279     public void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, @Nullable Uuid networkUuid,
2280                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2281                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2282             throws ExecutionException, InterruptedException {
2283         //Withdraw the corresponding routes from the BGP.
2284         //Get the network ID using the router ID.
2285         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2286                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2287         if (networkUuid == null) {
2288             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2289             return;
2290         }
2291
2292         if (externalIps.isEmpty()) {
2293             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2294             return;
2295         }
2296
2297         if (vpnName == null) {
2298             //Get the VPN Name using the network ID
2299             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2300             if (vpnName == null) {
2301                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2302                     networkUuid, routerId);
2303                 return;
2304             }
2305         }
2306         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2307
2308         //Remove custom FIB routes
2309         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2310         for (String extIp : externalIps) {
2311             extIp = NatUtil.validateAndAddNetworkMask(extIp);
2312             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2313         }
2314     }
2315
2316     protected void clrRtsFromBgpAndDelFibTs(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2317                                             final Uuid networkUuid, String extGwMacAddress,
2318                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2319             throws ExecutionException, InterruptedException {
2320         clearBgpRoutes(extIp, vpnName);
2321         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2322                 removeFlowInvTx);
2323     }
2324
2325     protected void delFibTsAndReverseTraffic(final Uint64 dpnId, String routerName, Uint32 routerId, String extIp,
2326                                              String vpnName, Uuid extNetworkId, Uint32 tempLabel,
2327                                              String gwMacAddress, boolean switchOver,
2328                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2329             throws ExecutionException, InterruptedException {
2330         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2331         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2332         if (routerName == null) {
2333             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2334             return;
2335         }
2336         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2337         if (extNwProvType == null) {
2338             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2339             return;
2340         }
2341         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2342          * external network provided type is VxLAN
2343          */
2344         if (extNwProvType == ProviderTypes.VXLAN) {
2345             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2346             );
2347             return;
2348         }
2349         if (tempLabel.longValue() < 0) {
2350             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2351                     extIp, routerId);
2352             return;
2353         }
2354         final Uint32 label = tempLabel;
2355         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2356         RemoveFibEntryInput input = null;
2357         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2358             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2359             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2360             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2361                     extRouter);
2362
2363             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2364                     externalSubnetId);
2365
2366             if (externalSubnet.isPresent()) {
2367                 vpnName =  externalSubnetId.getValue();
2368             }
2369         }
2370         final String externalVpn = vpnName;
2371         if (label != null && label.toJava() <= 0) {
2372             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2373                 extIp, routerId);
2374             input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2375                 .setSourceDpid(dpnId).setIpAddress(externalIp)
2376                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2377         } else {
2378             input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2379                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2380                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2381             removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2382             removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2383         }
2384         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2385
2386         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2387         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2388         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2389             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2390             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2391         }
2392         if (!switchOver) {
2393             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2394                 Futures.transformAsync(future, result -> {
2395                     //Release label
2396                     if (result.isSuccessful() && label != null && label.toJava() > 0) {
2397                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2398                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2399                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2400                         Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
2401                         if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
2402                             String errMsg = String.format(
2403                                     "ExternalRoutersListener: RPC call to remove VPN label "
2404                                             + "on dpn %s for prefix %s failed for vpn %s - %s",
2405                                     dpnId, externalIp, result.getErrors());
2406                             LOG.error(errMsg);
2407                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2408                         }
2409                         return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2410                     } else {
2411                         String errMsg =
2412                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2413                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2414                         LOG.error(errMsg);
2415                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2416                     }
2417                 }, MoreExecutors.directExecutor());
2418
2419             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2420
2421                 @Override
2422                 public void onFailure(@NonNull Throwable error) {
2423                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2424                         + "got external ip {}", label, extIp, error);
2425                 }
2426
2427                 @Override
2428                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2429                     if (result.isSuccessful()) {
2430                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2431                             + "from VPN {}", externalIp, externalVpn);
2432                     } else {
2433                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2434                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2435                     }
2436                 }
2437             }, MoreExecutors.directExecutor());
2438         } else {
2439             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2440                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2441         }
2442     }
2443
2444     private void delFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, String extIp, final String vpnName,
2445                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2446                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2447             throws ExecutionException, InterruptedException {
2448         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2449         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2450         if (routerName == null) {
2451             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2452             return;
2453         }
2454         //Get the external network provider type from networkId
2455         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2456         if (extNwProvType == null) {
2457             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2458                     networkUuid);
2459             return;
2460         }
2461         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2462          *  external network provided type is VxLAN
2463          */
2464         if (extNwProvType == ProviderTypes.VXLAN) {
2465             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2466             return;
2467         }
2468         //Get IPMaps from the DB for the router ID
2469         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2470         if (dbIpMaps.isEmpty()) {
2471             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2472             return;
2473         }
2474
2475         Uint32 tempLabel = NatConstants.INVALID_ID;
2476         for (IpMap dbIpMap : dbIpMaps) {
2477             String dbExternalIp = dbIpMap.getExternalIp();
2478             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2479             //Select the IPMap, whose external IP is the IP for which FIB is installed
2480             if (extIp.equals(dbExternalIp)) {
2481                 tempLabel = dbIpMap.getLabel();
2482                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2483                     tempLabel, dbExternalIp, routerId);
2484                 break;
2485             }
2486         }
2487         if (tempLabel == NatConstants.INVALID_ID) {
2488             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2489                     extIp, routerId);
2490             return;
2491         }
2492
2493         final Uint32 label = tempLabel;
2494         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2495         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2496                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2497                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2498         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2499
2500         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2501         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2502         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2503             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2504             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2505         }
2506         if (!switchOver) {
2507             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2508                     Futures.transformAsync(future, result -> {
2509                         //Release label
2510                         if (result.isSuccessful()) {
2511                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2512                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2513                             Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService
2514                                     .removeVpnLabel(labelInput);
2515                             if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
2516                                 String errMsg = String.format(
2517                                         "RPC call to remove VPN label on dpn %s for prefix %s "
2518                                                 + "failed for vpn %s - %s", dpnId, externalIp, vpnName,
2519                                         result.getErrors());
2520                                 LOG.error(errMsg);
2521                                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2522                             }
2523                             return JdkFutureAdapters.listenInPoolThread(labelFuture1);
2524                         } else {
2525                             String errMsg =
2526                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2527                                             + "prefix %s Failed - %s",
2528                                             dpnId, externalIp, result.getErrors());
2529                             LOG.error(errMsg);
2530                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2531                         }
2532                     }, MoreExecutors.directExecutor());
2533
2534             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2535
2536                 @Override
2537                 public void onFailure(@NonNull Throwable error) {
2538                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2539                 }
2540
2541                 @Override
2542                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2543                     if (result.isSuccessful()) {
2544                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2545                             + "from VPN {}", externalIp, vpnName);
2546                     } else {
2547                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2548                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2549                     }
2550                 }
2551             }, MoreExecutors.directExecutor());
2552         } else {
2553             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2554                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2555         }
2556     }
2557
2558     protected void clearFibTsAndReverseTraffic(final Uint64 dpnId, Uint32 routerId, Uuid networkUuid,
2559             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2560             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2561         //Withdraw the corresponding routes from the BGP.
2562         //Get the network ID using the router ID.
2563         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2564                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2565         if (networkUuid == null) {
2566             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2567             return;
2568         }
2569
2570         if (externalIps == null || externalIps.isEmpty()) {
2571             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2572             return;
2573         }
2574
2575         if (vpnName == null) {
2576             //Get the VPN Name using the network ID
2577             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2578             if (vpnName == null) {
2579                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2580                     networkUuid, routerId);
2581                 return;
2582             }
2583         }
2584         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2585
2586         //Remove custom FIB routes
2587         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2588         for (String extIp : externalIps) {
2589             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2590                     writeFlowInvTx);
2591         }
2592     }
2593
2594     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2595         //Inform BGP about the route removal
2596         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2597         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2598         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName);
2599         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), externalIp);
2600     }
2601
2602     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
2603             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2604         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2605         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2606             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2607         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2608     }
2609
2610     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
2611             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2612         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2613         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2614         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2615         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2616     }
2617
2618     /**
2619      * router association to vpn.
2620      *
2621      * @param routerName - Name of router
2622      * @param routerId - router id
2623      * @param bgpVpnName BGP VPN name
2624      */
2625     public void changeLocalVpnIdToBgpVpnId(String routerName, Uint32 routerId, String extNetwork, String bgpVpnName,
2626             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2627         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2628         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2629             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2630
2631             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2632
2633             if (bgpVpnId != NatConstants.INVALID_ID) {
2634                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2635                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2636                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2637                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2638                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2639                     getRoutersIdentifier(bgpVpnId), rtrs);
2640
2641                 // Get the allocated Primary NAPT Switch for this router
2642                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2643                         routerId, bgpVpnId);
2644                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2645
2646                 // Get the group ID
2647                 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2648                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, new Uuid(extNetwork),
2649                     true, writeFlowInvTx, extNwProvType);
2650             }
2651         }
2652     }
2653
2654     /**
2655      * router disassociation from vpn.
2656      *
2657      * @param routerName - Name of router
2658      * @param routerId - router id
2659      * @param bgpVpnName BGP VPN name
2660      */
2661     public void changeBgpVpnIdToLocalVpnId(String routerName, Uint32 routerId, String bgpVpnName, String extNetwork,
2662             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2663         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2664         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2665             Uint32 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2666             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2667
2668             // Get the allocated Primary NAPT Switch for this router
2669             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2670
2671             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2672             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2673
2674             // Get the group ID
2675             Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2676             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId,
2677                 new Uuid(extNetwork), true, writeFlowInvTx, extNwProvType);
2678         }
2679     }
2680
2681     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2682         InstanceIdentifier<Routers> routerInstanceIndentifier =
2683             InstanceIdentifier.builder(ExtRouters.class)
2684                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2685         try {
2686             Optional<Routers> routerData = SingleTransactionDataBroker
2687                     .syncReadOptional(dataBroker,
2688                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2689             return routerData.isPresent() && routerData.get().isEnableSnat();
2690         } catch (ReadFailedException e) {
2691             LOG.error("Failed to read data for router id {}", routerUuid, e);
2692             return false;
2693         }
2694     }
2695
2696     public void installFlowsWithUpdatedVpnId(Uint64 primarySwitchId, String routerName, Uint32 bgpVpnId,
2697                                              Uint32 routerId, Uuid extNwUuid, boolean isSnatCfgd,
2698                                              TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2699
2700         Uint32 changedVpnId = bgpVpnId;
2701         String idType = "BGP VPN";
2702         if (bgpVpnId == NatConstants.INVALID_ID) {
2703             changedVpnId = routerId;
2704             idType = "router";
2705         }
2706
2707         List<Uint64> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2708         if (switches.isEmpty()) {
2709             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2710             return;
2711         }
2712         for (Uint64 dpnId : switches) {
2713             // Update the BGP VPN ID in the SNAT miss entry to group
2714             if (!dpnId.equals(primarySwitchId)) {
2715                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2716                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2717                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2718                 Uint32 groupId = NatUtil.getUniqueId(idManager, NatConstants.SNAT_IDPOOL_NAME,
2719                     NatUtil.getGroupIdKey(routerName));
2720                 if (groupId != NatConstants.INVALID_ID) {
2721                     if (!isSnatCfgd) {
2722                         installGroup(dpnId, routerName, groupId, bucketInfoForNonNaptSwitches);
2723                     }
2724                     LOG.debug(
2725                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2726                             + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2727                     FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName,
2728                         groupId, changedVpnId);
2729                     mdsalManager.addFlow(confTx, flowEntity);
2730                 } else {
2731                     LOG.error("installFlowsWithUpdatedVpnId: Unable to get groupId for router:{}", routerName);
2732                 }
2733             } else {
2734                 LOG.debug(
2735                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2736                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2737                 FlowEntity flowEntity =
2738                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2739                 mdsalManager.addFlow(confTx, flowEntity);
2740
2741                 LOG.debug(
2742                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2743                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2744                         idType, changedVpnId, primarySwitchId);
2745                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2746                         changedVpnId, confTx, extNwProvType);
2747
2748                 LOG.debug(
2749                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2750                                 + "which punts the packet to the controller in the Primary switch {}",
2751                         idType, changedVpnId, primarySwitchId);
2752                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2753
2754                 LOG.debug(
2755                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2756                                 + " outgoing packet to FIB Table in the Primary switch {}",
2757                         idType, changedVpnId, primarySwitchId);
2758                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2759
2760                 LOG.debug(
2761                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2762                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2763                                 + " {}", idType, changedVpnId, primarySwitchId);
2764                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2765
2766                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2767                         primarySwitchId);
2768                 //Get the VPN ID from the ExternalNetworks model
2769                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, extNwUuid);
2770                 if (vpnUuid != null) {
2771                     Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2772                     //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table
2773                     // matching on the VPN ID.
2774                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
2775                         installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2776                     }
2777                 } else {
2778                     LOG.error("NAT Service : vpnUuid is null");
2779                 }
2780             }
2781         }
2782     }
2783
2784     public void updateNaptFlowsWithVpnId(Uint64 dpnId, String routerName, Uint32 routerId, Uint32 bgpVpnId) {
2785         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2786         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2787         if (ipPortMapping == null) {
2788             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2789             return;
2790         }
2791         // Get the External Gateway MAC Address
2792         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2793         if (extGwMacAddress != null) {
2794             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2795                     extGwMacAddress, routerId);
2796         } else {
2797             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2798                     routerId);
2799             return;
2800         }
2801         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2802             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2803                 String ipPortInternal = ipPortMap.getIpPortInternal();
2804                 String[] ipPortParts = ipPortInternal.split(":");
2805                 if (ipPortParts.length != 2) {
2806                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2807                     return;
2808                 }
2809                 String internalIp = ipPortParts[0];
2810                 String internalPort = ipPortParts[1];
2811                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2812                         internalIp, internalPort);
2813                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2814                 NAPTEntryEvent.Protocol protocol;
2815                 switch (protocolTypes) {
2816                     case TCP:
2817                         protocol = NAPTEntryEvent.Protocol.TCP;
2818                         break;
2819                     case UDP:
2820                         protocol = NAPTEntryEvent.Protocol.UDP;
2821                         break;
2822                     default:
2823                         protocol = NAPTEntryEvent.Protocol.TCP;
2824                 }
2825                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2826                 SessionAddress externalAddress =
2827                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2828                 Uint32 internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2829                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2830                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2831                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2832                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2833             }
2834         }
2835     }
2836
2837     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName, Uint32 groupId,
2838                                                           Uint32 changedVpnId) {
2839
2840         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2841             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2842         List<MatchInfo> matches = new ArrayList<>();
2843         matches.add(MatchEthernetType.IPV4);
2844         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2845                 MetaDataUtil.METADATA_MASK_VRFID));
2846
2847         List<ActionInfo> actionsInfo = new ArrayList<>();
2848         Uint64 tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2849                 elanManager, idManager, changedVpnId, routerName);
2850         actionsInfo.add(new ActionSetFieldTunnelId(tunnelId));
2851         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2852                 actionsInfo);
2853         actionsInfo.add(new ActionGroup(groupId.longValue()));
2854         List<InstructionInfo> instructions = new ArrayList<>();
2855         instructions.add(new InstructionApplyActions(actionsInfo));
2856         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2857         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2858             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2859             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2860
2861         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2862         return flowEntity;
2863     }
2864
2865     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(Uint64 dpId, String routerName,
2866                                                                         Uint32 changedVpnId) {
2867
2868         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2869                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2870         List<MatchInfo> matches = new ArrayList<>();
2871         matches.add(MatchEthernetType.IPV4);
2872         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2873                 MetaDataUtil.METADATA_MASK_VRFID));
2874
2875         List<InstructionInfo> instructions = new ArrayList<>();
2876         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2877
2878         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2879         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2880             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2881             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2882
2883         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2884         return flowEntity;
2885     }
2886
2887     // TODO : Replace this with ITM Rpc once its available with full functionality
2888     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(Uint64 dpnId, String routerName,
2889                                                                      Uint32 routerId, Uint32 changedVpnId,
2890                                                                      TypedWriteTransaction<Configuration> confTx,
2891                                                                      ProviderTypes extNwProvType) {
2892
2893         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2894             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2895         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2896                 extNwProvType);
2897         mdsalManager.addFlow(confTx, flowEntity);
2898     }
2899
2900     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(Uint64 dpId, String routerName,
2901                                                          Uint32 routerIdLongVal, Uint32 changedVpnId,
2902                                                          ProviderTypes extNwProvType) {
2903         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2904             dpId, routerName, changedVpnId);
2905         List<MatchInfo> matches = new ArrayList<>();
2906         matches.add(MatchEthernetType.IPV4);
2907
2908         Uint64 tunnelId = Uint64.valueOf(changedVpnId);
2909         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2910             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2911         }
2912         matches.add(new MatchTunnelId(tunnelId));
2913
2914         List<InstructionInfo> instructions = new ArrayList<>();
2915         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2916             MetaDataUtil.METADATA_MASK_VRFID));
2917         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2918         Uint32 routerId = routerIdLongVal;
2919         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId);
2920         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2921             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2922             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2923         return flowEntity;
2924     }
2925
2926     public void createOutboundTblEntryWithBgpVpn(Uint64 dpnId, Uint32 routerId, Uint32 changedVpnId,
2927                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2928         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2929             dpnId, routerId, changedVpnId);
2930         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2931                 NwConstants.IP_PROT_TCP);
2932         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2933         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2934
2935         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2936                 NwConstants.IP_PROT_UDP);
2937         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2938         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2939
2940         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2941         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2942         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2943     }
2944
2945     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(Uint64 dpId, Uint32 routerId,
2946                                                            Uint32 changedVpnId, int protocol) {
2947         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2948             dpId, routerId, changedVpnId);
2949         Uint64 cookie = getCookieOutboundFlow(routerId);
2950         List<MatchInfo> matches = new ArrayList<>();
2951         matches.add(MatchEthernetType.IPV4);
2952         matches.add(new MatchIpProtocol((short)protocol));
2953         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2954                 MetaDataUtil.METADATA_MASK_VRFID));
2955
2956         List<InstructionInfo> instructions = new ArrayList<>();
2957         List<ActionInfo> actionsInfos = new ArrayList<>();
2958         actionsInfos.add(new ActionPuntToController());
2959         if (snatPuntTimeout != 0) {
2960             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2961         }
2962         instructions.add(new InstructionApplyActions(actionsInfos));
2963
2964         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2965         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2966             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2967         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2968         return flowEntity;
2969     }
2970
2971     public void installNaptPfibEntryWithBgpVpn(Uint64 dpnId, Uint32 segmentId, Uint32 changedVpnId,
2972                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2973         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2974             dpnId, segmentId, changedVpnId);
2975         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2976         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2977     }
2978
2979     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(Uint64 dpId, Uint32 segmentId, Uint32 changedVpnId) {
2980
2981         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2982             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2983         List<MatchInfo> matches = new ArrayList<>();
2984         matches.add(MatchEthernetType.IPV4);
2985         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId.longValue()),
2986                 MetaDataUtil.METADATA_MASK_VRFID));
2987
2988         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2989         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2990         listActionInfo.add(new ActionNxLoadInPort(Uint64.valueOf(BigInteger.ZERO)));
2991         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2992         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2993
2994         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2995         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2996             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2997             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2998         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2999         return flowEntity;
3000     }
3001
3002     @Override
3003     protected ExternalRoutersListener getDataTreeChangeListener() {
3004         return ExternalRoutersListener.this;
3005     }
3006
3007     protected void installNaptPfibEntriesForExternalSubnets(String routerName, Uint64 dpnId,
3008                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
3009         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
3010                 routerName);
3011         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
3012             Uint32 subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
3013             if (subnetVpnId != NatConstants.INVALID_ID) {
3014                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
3015                     + "and vpnId {}", dpnId, subnetVpnId);
3016                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
3017             }
3018         }
3019     }
3020 }