natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.Nonnull;
  35 import javax.annotation.Nullable;
  36 import javax.annotation.PostConstruct;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.infra.Datastore.Configuration;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  49 import org.opendaylight.genius.mdsalutil.ActionInfo;
  50 import org.opendaylight.genius.mdsalutil.BucketInfo;
  51 import org.opendaylight.genius.mdsalutil.FlowEntity;
  52 import org.opendaylight.genius.mdsalutil.GroupEntity;
  53 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  54 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  55 import org.opendaylight.genius.mdsalutil.MatchInfo;
  56 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  59 import org.opendaylight.genius.mdsalutil.UpgradeState;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  85 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  86 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 148 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 149 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 150 import org.opendaylight.yangtools.yang.binding.DataObject;
 151 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 152 import org.opendaylight.yangtools.yang.common.RpcResult;
 153 import org.slf4j.Logger;
 154 import org.slf4j.LoggerFactory;
 155
 156 @Singleton
 157 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 158     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 159
 160     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 161     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 162
 163     private final DataBroker dataBroker;
 164     private final ManagedNewTransactionRunner txRunner;
 165     private final IMdsalApiManager mdsalManager;
 166     private final ItmRpcService itmManager;
 167     private final OdlInterfaceRpcService odlInterfaceRpcService;
 168     private final IdManagerService idManager;
 169     private final NaptManager naptManager;
 170     private final NAPTSwitchSelector naptSwitchSelector;
 171     private final IBgpManager bgpManager;
 172     private final VpnRpcService vpnService;
 173     private final FibRpcService fibService;
 174     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 175     private final NaptEventHandler naptEventHandler;
 176     private final NaptPacketInHandler naptPacketInHandler;
 177     private final IFibManager fibManager;
 178     private final IVpnManager vpnManager;
 179     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 180     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 181     private final NatMode natMode;
 182     private final INeutronVpnManager nvpnManager;
 183     private final IElanService elanManager;
 184     private final JobCoordinator coordinator;
 185     private final UpgradeState upgradeState;
 186     private final IInterfaceManager interfaceManager;
 187     private final int snatPuntTimeout;
 188
 189     @Inject
 190     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 191                                    final ItmRpcService itmManager,
 192                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 193                                    final IdManagerService idManager,
 194                                    final NaptManager naptManager,
 195                                    final NAPTSwitchSelector naptSwitchSelector,
 196                                    final IBgpManager bgpManager,
 197                                    final VpnRpcService vpnService,
 198                                    final FibRpcService fibService,
 199                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 200                                    final NaptEventHandler naptEventHandler,
 201                                    final NaptPacketInHandler naptPacketInHandler,
 202                                    final IFibManager fibManager,
 203                                    final IVpnManager vpnManager,
 204                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 205                                    final INeutronVpnManager nvpnManager,
 206                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 207                                    final NatserviceConfig config,
 208                                    final IElanService elanManager,
 209                                    final JobCoordinator coordinator,
 210                                    final UpgradeState upgradeState,
 211                                    final IInterfaceManager interfaceManager) {
 212         super(Routers.class, ExternalRoutersListener.class);
 213         this.dataBroker = dataBroker;
 214         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 215         this.mdsalManager = mdsalManager;
 216         this.itmManager = itmManager;
 217         this.odlInterfaceRpcService = odlInterfaceRpcService;
 218         this.idManager = idManager;
 219         this.naptManager = naptManager;
 220         this.naptSwitchSelector = naptSwitchSelector;
 221         this.bgpManager = bgpManager;
 222         this.vpnService = vpnService;
 223         this.fibService = fibService;
 224         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 225         this.naptEventHandler = naptEventHandler;
 226         this.naptPacketInHandler = naptPacketInHandler;
 227         this.fibManager = fibManager;
 228         this.vpnManager = vpnManager;
 229         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 230         this.nvpnManager = nvpnManager;
 231         this.elanManager = elanManager;
 232         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 233         this.coordinator = coordinator;
 234         this.upgradeState = upgradeState;
 235         this.interfaceManager = interfaceManager;
 236         if (config != null) {
 237             this.natMode = config.getNatMode();
 238             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 239         } else {
 240             this.natMode = NatMode.Controller;
 241             this.snatPuntTimeout = 0;
 242         }
 243     }
 244
 245     @Override
 246     @PostConstruct
 247     public void init() {
 248         LOG.info("{} init", getClass().getSimpleName());
 249         // This class handles ExternalRouters for Controller SNAT mode.
 250         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 251         if (natMode == NatMode.Controller) {
 252             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 253             NatUtil.createGroupIdPool(idManager);
 254         }
 255     }
 256
 257     @Override
 258     protected InstanceIdentifier<Routers> getWildCardPath() {
 259         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 260     }
 261
 262     @Override
 263     // TODO Clean up the exception handling
 264     @SuppressWarnings("checkstyle:IllegalCatch")
 265     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 266         // Populate the router-id-name container
 267         String routerName = routers.getRouterName();
 268         LOG.info("add : external router event for {}", routerName);
 269         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 270         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 271         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 272         try {
 273             coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 274                 () -> Collections.singletonList(
 275                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 276                         LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 277                         long bgpVpnId = NatConstants.INVALID_ID;
 278                         if (bgpVpnUuid != null) {
 279                             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 280                         }
 281                         addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 282                         // Allocate Primary Napt Switch for this router
 283                         BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 284                         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 285                             if (!routers.isEnableSnat()) {
 286                                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 287                                 /* If SNAT is disabled on ext-router though L3_FIB_TABLE(21) -> PSNAT_TABLE(26) flow
 288                                  * is required for DNAT. Hence writeFlowInvTx object submit is required.
 289                                  */
 290                                 return;
 291                             }
 292                             handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 293                         }
 294                     })), NatConstants.NAT_DJC_MAX_RETRIES);
 295         } catch (Exception ex) {
 296             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 297                     routerName, ex);
 298         }
 299     }
 300
 301     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 302                                  TypedWriteTransaction<Configuration> confTx) {
 303         String routerName = routers.getRouterName();
 304         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 305
 306         naptManager.initialiseExternalCounter(routers, routerId);
 307         subnetRegisterMapping(routers, routerId);
 308
 309         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 310             primarySwitchId, routerName);
 311
 312         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 313                 routers.getNetworkId());
 314         if (extNwProvType == null) {
 315             LOG.error("handleEnableSnat : External Network Provider Type missing");
 316             return;
 317         }
 318
 319         if (bgpVpnId != NatConstants.INVALID_ID) {
 320             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 321                     extNwProvType);
 322         } else {
 323             // write metadata and punt
 324             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 325             // Now install entries in SNAT tables to point to Primary for each router
 326             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 327             for (BigInteger dpnId : switches) {
 328                 // Handle switches and NAPT switches separately
 329                 if (!dpnId.equals(primarySwitchId)) {
 330                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 331                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 332                 } else {
 333                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 334                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, confTx);
 335                 }
 336             }
 337         }
 338
 339         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 340         if (externalIps.isEmpty()) {
 341             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 342             return;
 343         } else {
 344             for (String externalIpAddrPrefix : externalIps) {
 345                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 346                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 347                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 348                 );
 349             }
 350         }
 351         LOG.debug("handleEnableSnat : Exit");
 352     }
 353
 354     private BigInteger getPrimaryNaptSwitch(String routerName) {
 355         // Allocate Primary Napt Switch for this router
 356         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 357         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 358             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 359                 primarySwitchId, routerName);
 360             return primarySwitchId;
 361         }
 362         // Validating and creating VNI pool during when NAPT switch is selected.
 363         // With Assumption this might be the first NAT service comes up.
 364         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 365                 NatConstants.ODL_VNI_POOL_NAME);
 366         // Allocated an id from VNI pool for the Router.
 367         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 368         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 369         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 370
 371         return primarySwitchId;
 372     }
 373
 374     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 375                                                      TypedWriteTransaction<Configuration> confTx) {
 376         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 377         if (extVpnId == NatConstants.INVALID_ID) {
 378             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 379             extVpnId = routerId;
 380         }
 381         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 382         if (externalIps.isEmpty()) {
 383             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 384                 dpnId, extVpnId);
 385             return;
 386         }
 387         for (String ip : externalIps) {
 388             Uuid subnetId = getSubnetIdForFixedIp(ip);
 389             if (subnetId != null) {
 390                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 391                 if (subnetVpnId != NatConstants.INVALID_ID) {
 392                     extVpnId = subnetVpnId;
 393                 }
 394                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 395                     dpnId, extVpnId, subnetId);
 396                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 397                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 398             }
 399         }
 400     }
 401
 402     @Nullable
 403     private Uuid getSubnetIdForFixedIp(String ip) {
 404         if (ip != null) {
 405             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 406             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 407             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 408         }
 409         LOG.error("getSubnetIdForFixedIp : ip is null");
 410         return null;
 411     }
 412
 413     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 414         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 415         List<Uuid> subnetList = routerEntry.getSubnetIds();
 416         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 417         int counter = 0;
 418         int extIpCounter = externalIps.size();
 419         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 420                 counter, extIpCounter);
 421         for (Uuid subnet : subnetList) {
 422             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 423             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 424                 .builder(Subnetmaps.class)
 425                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 426                 .build();
 427             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 428             if (sn.isPresent()) {
 429                 // subnets
 430                 Subnetmap subnetmapEntry = sn.get();
 431                 String subnetString = subnetmapEntry.getSubnetIp();
 432                 String[] subnetSplit = subnetString.split("/");
 433                 String subnetIp = subnetSplit[0];
 434                 try {
 435                     InetAddress address = InetAddress.getByName(subnetIp);
 436                     if (address instanceof Inet6Address) {
 437                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 438                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 439                         continue;
 440                     }
 441                 } catch (UnknownHostException e) {
 442                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 443                     return;
 444                 }
 445                 String subnetPrefix = "0";
 446                 if (subnetSplit.length == 2) {
 447                     subnetPrefix = subnetSplit[1];
 448                 }
 449                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 450                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 451                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 452                 //externalIps
 453                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 454                         counter, extIpCounter);
 455                 if (extIpCounter != 0) {
 456                     if (counter < extIpCounter) {
 457                         String[] ipSplit = externalIps.get(counter).split("/");
 458                         String externalIp = ipSplit[0];
 459                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 460                         if (ipSplit.length == 2) {
 461                             extPrefix = ipSplit[1];
 462                         }
 463                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 464                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 465                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 466                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 467                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 468                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 469                     } else {
 470                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 471                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 472                         String[] ipSplit = externalIps.get(counter).split("/");
 473                         String externalIp = ipSplit[0];
 474                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 475                         if (ipSplit.length == 2) {
 476                             extPrefix = ipSplit[1];
 477                         }
 478                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 479                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 480                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 481                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 482                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 483                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 484                             externalIp, extPrefix);
 485                     }
 486                 }
 487                 counter++;
 488                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 489             } else {
 490                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 491             }
 492         }
 493     }
 494
 495     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 496             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx) {
 497         //Check if BGP VPN exists. If exists then invoke the new method.
 498         if (bgpVpnId != NatConstants.INVALID_ID) {
 499             if (bgpVpnUuid != null) {
 500                 String bgpVpnName = bgpVpnUuid.getValue();
 501                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 502                     bgpVpnId, bgpVpnName);
 503                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 504                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 505                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 506             }
 507             if (create) {
 508                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 509             } else {
 510                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 511             }
 512             return;
 513         }
 514
 515         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 516         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 517     }
 518
 519     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 520             TypedReadWriteTransaction<Configuration> confTx) {
 521         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 522         if (switches.isEmpty()) {
 523             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 524             return;
 525         }
 526         if (routerId == NatConstants.INVALID_ID) {
 527             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 528                     + "default NAT route in FIB", routerName);
 529             return;
 530         }
 531         for (BigInteger dpnId : switches) {
 532             if (create) {
 533                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 534                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 535                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 536             } else {
 537                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 538                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 539                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 540             }
 541         }
 542     }
 543
 544     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 545         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 546         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 547         if (dpnIds.isEmpty()) {
 548             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 549                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 550             return;
 551         }
 552         for (BigInteger dpnId : dpnIds) {
 553             if (bgpVpnId != NatConstants.INVALID_ID) {
 554                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 555                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 556                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 557             } else {
 558                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 559                     + "in dpn {} for the internal vpn", routerId, dpnId);
 560                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 561             }
 562         }
 563     }
 564
 565     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 566         long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx) {
 567         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 568         if (dpnIds.isEmpty()) {
 569             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 570                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 571             return;
 572         }
 573         for (BigInteger dpnId : dpnIds) {
 574             if (bgpVpnId != NatConstants.INVALID_ID) {
 575                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 576                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 577                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 578             } else {
 579                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 580                     + "in dpn {} for the internal vpn", routerId, dpnId);
 581                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 582             }
 583         }
 584     }
 585
 586     // TODO Clean up the exception handling
 587     @SuppressWarnings("checkstyle:IllegalCatch")
 588     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 589                                                           InstanceIdentifier<T> path) {
 590         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 591
 592         try {
 593             return tx.read(datastoreType, path).get();
 594         } catch (Exception e) {
 595             throw new RuntimeException(e);
 596         }
 597     }
 598
 599     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 600         TypedWriteTransaction<Configuration> confTx) {
 601         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 602         if (routerId != NatConstants.INVALID_ID) {
 603             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 604                     primarySwitchId, routerId);
 605             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 606         } else {
 607             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 608                 + "createAndInstallMissEntry", routerName);
 609         }
 610     }
 611
 612     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 613         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 614                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 615     }
 616
 617     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 618         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 619                 .FLOWID_SEPARATOR + vpnId;
 620     }
 621
 622     public BigInteger getCookieOutboundFlow(long routerId) {
 623         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 624             BigInteger.valueOf(routerId));
 625     }
 626
 627     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 628         long l4SrcPortField;
 629         long l4DstPortField;
 630         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 631
 632         if (protocol == NwConstants.IP_PROT_TCP) {
 633             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 634             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 635         } else {
 636             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 637             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 638         }
 639         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 640                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 641                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 642                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 643                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 644                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 645                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 646                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 647                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 648                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 649                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 650                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 651                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 652                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 653                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 654
 655         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 656                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 657     }
 658
 659     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 660         List<MatchInfo> matches = new ArrayList<>();
 661         matches.add(MatchEthernetType.IPV4);
 662         matches.add(MatchIpProtocol.ICMP);
 663         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 664
 665         List<ActionInfo> actionInfos = new ArrayList<>();
 666         actionInfos.add(new ActionDrop());
 667
 668         List<InstructionInfo> instructions = new ArrayList<>();
 669         instructions.add(new InstructionApplyActions(actionInfos));
 670
 671         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 672                 NwConstants.IP_PROT_ICMP);
 673         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 674                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 675                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 676         return flow;
 677     }
 678
 679     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 680         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 681         BigInteger cookie = getCookieOutboundFlow(routerId);
 682         List<MatchInfo> matches = new ArrayList<>();
 683         matches.add(MatchEthernetType.IPV4);
 684         matches.add(new MatchIpProtocol((short)protocol));
 685         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 686
 687         List<InstructionInfo> instructions = new ArrayList<>();
 688         List<ActionInfo> actionsInfos = new ArrayList<>();
 689         actionsInfos.add(new ActionPuntToController());
 690         if (snatPuntTimeout != 0) {
 691             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 692         }
 693         instructions.add(new InstructionApplyActions(actionsInfos));
 694
 695         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 696         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 697             5, flowRef, 0, 0,
 698             cookie, matches, instructions);
 699         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 700         return flowEntity;
 701     }
 702
 703     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 704         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 705         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 706         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 707         mdsalManager.addFlow(confTx, tcpflowEntity);
 708
 709         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 710         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 711         mdsalManager.addFlow(confTx, udpflowEntity);
 712
 713         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 714         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 715         mdsalManager.addFlow(confTx, icmpDropFlow);
 716     }
 717
 718     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 719         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 720         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 721         try {
 722             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 723                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 724                     .setSourceDpid(srcDpId)
 725                     .setDestinationDpid(dstDpId)
 726                     .setTunnelType(tunType)
 727                     .build());
 728             rpcResult = result.get();
 729             if (!rpcResult.isSuccessful()) {
 730                 tunType = TunnelTypeGre.class;
 731                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 732                     .setSourceDpid(srcDpId)
 733                     .setDestinationDpid(dstDpId)
 734                     .setTunnelType(tunType)
 735                     .build());
 736                 rpcResult = result.get();
 737                 if (!rpcResult.isSuccessful()) {
 738                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 739                             rpcResult.getErrors());
 740                 } else {
 741                     return rpcResult.getResult().getInterfaceName();
 742                 }
 743                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 744                         rpcResult.getErrors());
 745             } else {
 746                 return rpcResult.getResult().getInterfaceName();
 747             }
 748         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 749             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 750                     + "between {} and {}", srcDpId, dstDpId, e);
 751         }
 752
 753         return null;
 754     }
 755
 756     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 757         TypedWriteTransaction<Configuration> confTx) {
 758
 759         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 760         // Install miss entry pointing to group
 761         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 762         mdsalManager.addFlow(confTx, flowEntity);
 763     }
 764
 765     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 766             String routerName, long routerId) {
 767         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 768             dpnId, bucketInfo.get(0));
 769         // Install the select group
 770         long groupId = createGroupId(getGroupIdKey(routerName));
 771         GroupEntity groupEntity =
 772             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 773         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 774         mdsalManager.syncInstallGroup(groupEntity);
 775         // Install miss entry pointing to group
 776         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 777         if (flowEntity == null) {
 778             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 779                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 780                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 781             return;
 782         }
 783         mdsalManager.installFlow(flowEntity);
 784     }
 785
 786     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 787         long groupId = createGroupId(getGroupIdKey(routerName));
 788         GroupEntity groupEntity =
 789             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 790         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 791         mdsalManager.syncInstallGroup(groupEntity);
 792         return groupId;
 793     }
 794
 795     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 796         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 797                 dpId, routerName, groupId);
 798         List<MatchInfo> matches = new ArrayList<>();
 799         matches.add(MatchEthernetType.IPV4);
 800         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 801
 802         List<ActionInfo> actionsInfo = new ArrayList<>();
 803         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 804                 routerName);
 805         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 806         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 807         actionsInfo.add(new ActionGroup(groupId));
 808         List<InstructionInfo> instructions = new ArrayList<>();
 809         instructions.add(new InstructionApplyActions(actionsInfo));
 810         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 811         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 812                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 813                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 814
 815         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 816         return flowEntity;
 817     }
 818
 819     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 820
 821         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 822             routerName);
 823         List<MatchInfo> matches = new ArrayList<>();
 824         matches.add(MatchEthernetType.IPV4);
 825         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 826
 827         List<InstructionInfo> instructions = new ArrayList<>();
 828         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 829
 830         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 831         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 832             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 833             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 834
 835         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 836         return flowEntity;
 837     }
 838
 839     // TODO : Replace this with ITM Rpc once its available with full functionality
 840     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 841         TypedWriteTransaction<Configuration> confTx) {
 842
 843         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 844             dpnId, routerName);
 845         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 846         if (flowEntity == null) {
 847             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 848                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 849                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 850                     dpnId, routerName);
 851             return;
 852         }
 853         mdsalManager.addFlow(confTx, flowEntity);
 854
 855     }
 856
 857     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 858         List<MatchInfo> matches = new ArrayList<>();
 859         matches.add(MatchEthernetType.IPV4);
 860         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 861                 routerName);
 862         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 863         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 864         List<InstructionInfo> instructions = new ArrayList<>();
 865         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 866                 MetaDataUtil.METADATA_MASK_VRFID));
 867         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 868         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 869                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 870                 instructions);
 871         return flowEntity;
 872     }
 873
 874     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 875         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 876                 .FLOWID_SEPARATOR + routerID;
 877     }
 878
 879     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 880         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 881                 .FLOWID_SEPARATOR + routerID;
 882     }
 883
 884     private String getGroupIdKey(String routerName) {
 885         return "snatmiss." + routerName;
 886     }
 887
 888     protected long createGroupId(String groupIdKey) {
 889         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 890             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 891             .build();
 892         try {
 893             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 894             RpcResult<AllocateIdOutput> rpcResult = result.get();
 895             return rpcResult.getResult().getIdValue();
 896         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 897             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 898         }
 899         return 0;
 900     }
 901
 902     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 903         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 904         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 905         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 906         List<BucketInfo> listBucketInfo = new ArrayList<>();
 907
 908         if (ifNamePrimary != null) {
 909             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 910             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 911                     interfaceManager, ifNamePrimary, routerId, true);
 912             if (listActionInfoPrimary.isEmpty()) {
 913                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 914                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 915                         ifNamePrimary);
 916             }
 917         } else {
 918             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 919                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 920         }
 921         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 922
 923         listBucketInfo.add(0, bucketPrimary);
 924         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 925     }
 926
 927     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 928                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 929         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 930         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 931         List<BucketInfo> listBucketInfo = new ArrayList<>();
 932
 933         if (ifNamePrimary != null) {
 934             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 935                     ifNamePrimary);
 936             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 937                     interfaceManager, ifNamePrimary, routerId, true);
 938             if (listActionInfoPrimary.isEmpty()) {
 939                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 940                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 941                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 942             }
 943         } else {
 944             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 945                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 946         }
 947         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 948
 949         listBucketInfo.add(0, bucketPrimary);
 950         return listBucketInfo;
 951     }
 952
 953     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 954         TypedWriteTransaction<Configuration> confTx) {
 955
 956        /*
 957         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 958         */
 959
 960         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 961
 962 /*
 963         List<BucketInfo> listBucketInfo = new ArrayList<>();
 964         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 965         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 966         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 967         listBucketInfo.add(0, bucketPrimary);
 968 */
 969
 970         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 971         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 972         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 973         installNaptPfibEntry(dpnId, routerId, confTx);
 974         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 975         if (networkId != null) {
 976             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 977             if (vpnUuid != null) {
 978                 Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 979                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 980                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 981                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 982                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 983                         installNaptPfibEntry(dpnId, vpnId, null);
 984                     }
 985                     return Collections.emptyList();
 986                 });
 987             } else {
 988                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 989             }
 990         } else {
 991             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 992         }
 993     }
 994
 995     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 996         List<BucketInfo> listBucketInfo = new ArrayList<>();
 997         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 998         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 999         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
1000         listBucketInfo.add(0, bucketPrimary);
1001         return listBucketInfo;
1002     }
1003
1004     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, TypedWriteTransaction<Configuration> confTx) {
1005         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
1006         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
1007         if (confTx != null) {
1008             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
1009         } else {
1010             mdsalManager.installFlow(naptPfibFlowEntity);
1011         }
1012     }
1013
1014     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
1015
1016         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
1017         List<MatchInfo> matches = new ArrayList<>();
1018         matches.add(MatchEthernetType.IPV4);
1019         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1020
1021         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1022         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1023         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1024         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1025         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1026
1027         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1028         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1029             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1030             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1031         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1032         return flowEntity;
1033     }
1034
1035     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1036         long routerId, String routerName, String externalIp) {
1037         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1038             dpnId, routerId, externalIp);
1039         Uuid networkId = router.getNetworkId();
1040         if (networkId == null) {
1041             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1042             return;
1043         }
1044         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1045         if (vpnName == null) {
1046             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1047                     + "configuration {} in router {}", networkId, externalIp, routerId);
1048             return;
1049         }
1050         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1051             externalIp, networkId, router, confTx);
1052         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1053             dpnId, routerId, externalIp);
1054     }
1055
1056     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1057                                                 final long routerId, final String routerName, final String externalIp,
1058                                                 final Uuid extNetworkId, final Routers router,
1059                                                 final TypedWriteTransaction<Configuration> confTx) {
1060         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1061                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1062         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1063         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1064         if (rd == null || rd.isEmpty()) {
1065             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1066             return;
1067         }
1068         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1069         if (extNwProvType == null) {
1070             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1071             return;
1072         }
1073         if (extNwProvType == ProviderTypes.VXLAN) {
1074             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1075                 nextHopIp, routerId, routerName, confTx);
1076             return;
1077         }
1078         //Generate VPN label for the external IP
1079         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1080             .setIpPrefix(externalIp).build();
1081         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1082
1083         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1084         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1085             if (result.isSuccessful()) {
1086                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1087                 GenerateVpnLabelOutput output = result.getResult();
1088                 final long label = output.getLabel();
1089
1090                 int externalIpInDsFlag = 0;
1091                 //Get IPMaps from the DB for the router ID
1092                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1093                 if (dbIpMaps != null) {
1094                     for (IpMap dbIpMap : dbIpMaps) {
1095                         String dbExternalIp = dbIpMap.getExternalIp();
1096                         //Select the IPMap, whose external IP is the IP for which FIB is installed
1097                         if (dbExternalIp.contains(externalIp)) {
1098                             String dbInternalIp = dbIpMap.getInternalIp();
1099                             IpMapKey dbIpMapKey = dbIpMap.key();
1100                             LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1101                                     + "and externalIp {}", label, dbInternalIp, externalIp);
1102                             IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1103                                 .setExternalIp(dbExternalIp).setLabel(label).build();
1104                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1105                                 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1106                             externalIpInDsFlag++;
1107                         }
1108                     }
1109                     if (externalIpInDsFlag <= 0) {
1110                         LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1111                                 + "Failed to update label {} for routerId {} in DS",
1112                                 externalIp, label, routerId);
1113                         String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1114                             + " found in DS for router %s", label, externalIp, routerId);
1115                         return Futures.immediateFailedFuture(new Exception(errMsg));
1116                     }
1117                 } else {
1118                     LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1119                             + " routerId {} in DS", label, externalIp, routerId);
1120                 }
1121                 //Inform BGP
1122                 long l3vni = 0;
1123                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1124                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1125                 }
1126                 Routers extRouter = router != null ? router :
1127                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1128                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1129                         extRouter);
1130                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1131                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1132                     RouteOrigin.STATIC, dpnId);
1133
1134                 //Install custom FIB routes
1135                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1136                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1137                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1138                         extNwProvType);
1139                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1140
1141                 //Install custom FIB routes - FIB table.
1142                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1143                         routerName, externalIp);
1144                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1145                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1146                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1147                 }
1148                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1149                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1150                         externalSubnetId);
1151                 String externalVpn = vpnName;
1152                 if (externalSubnet.isPresent()) {
1153                     externalVpn =  externalSubnetId.getValue();
1154                 }
1155                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1156                     .setVpnName(externalVpn)
1157                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1158                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1159                     .setInstruction(fibTableCustomInstructions).build();
1160                 return fibService.createFibEntry(input);
1161             } else {
1162                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1163                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1164                     externalIp, vpnName, result.getErrors());
1165                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1166             }
1167         }, MoreExecutors.directExecutor());
1168
1169         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1170
1171             @Override
1172             public void onFailure(@Nonnull Throwable error) {
1173                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1174             }
1175
1176             @Override
1177             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1178                 if (result.isSuccessful()) {
1179                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1180                             externalIp);
1181                 } else {
1182                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1183                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1184                 }
1185             }
1186         }, MoreExecutors.directExecutor());
1187     }
1188
1189     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1190             String externalIp) {
1191         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1192         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1193         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1194                 externalIp, router);
1195         int instructionIndex = 0;
1196         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1197             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1198             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1199                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1200             instructionIndex++;
1201         }
1202
1203         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1204         return fibTableCustomInstructions;
1205     }
1206
1207     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1208         TypedWriteTransaction<Configuration> confTx) {
1209         List<MatchInfo> matches = new ArrayList<>();
1210         matches.add(MatchEthernetType.MPLS_UNICAST);
1211         matches.add(new MatchMplsLabel(serviceId));
1212
1213         List<Instruction> instructions = new ArrayList<>();
1214         List<ActionInfo> actionsInfos = new ArrayList<>();
1215         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1216         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1217         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1218         instructions.add(writeInstruction);
1219         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1220
1221         // Install the flow entry in L3_LFIB_TABLE
1222         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1223
1224         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1225             10, flowRef, 0, 0,
1226             COOKIE_VM_LFIB_TABLE, matches, instructions);
1227
1228         mdsalManager.addFlow(confTx, dpId, flowEntity);
1229
1230         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1231     }
1232
1233     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1234         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1235         ProviderTypes extNwProvType) {
1236
1237         List<MatchInfo> mkMatches = new ArrayList<>();
1238
1239         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1240             dpnId, serviceId, customInstructions);
1241
1242         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1243             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1244         } else {
1245             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1246         }
1247
1248         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1249             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1250             String.format("%s:%d", "TST Flow Entry ", serviceId),
1251             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1252
1253         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1254     }
1255
1256     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1257         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1258             new RouterIdsKey(routerId)).build();
1259     }
1260
1261     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1262         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1263                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1264     }
1265
1266     @Override
1267     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1268         String routerName = original.getRouterName();
1269         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1270         if (routerId == NatConstants.INVALID_ID) {
1271             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1272             return;
1273         }
1274         // Check if its update on SNAT flag
1275         boolean originalSNATEnabled = original.isEnableSnat();
1276         boolean updatedSNATEnabled = update.isEnableSnat();
1277         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1278             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1279         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1280          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1281          */
1282         long bgpVpnId = NatConstants.INVALID_ID;
1283         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1284         if (bgpVpnUuid != null) {
1285             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1286         }
1287         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1288         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1289             // Router has no interface attached
1290             return;
1291         }
1292         final long finalBgpVpnId = bgpVpnId;
1293         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1294             List<ListenableFuture<Void>> futures = new ArrayList<>();
1295             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1296                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1297                     Uuid networkId = original.getNetworkId();
1298                     if (originalSNATEnabled != updatedSNATEnabled) {
1299                         if (originalSNATEnabled) {
1300                             //SNAT disabled for the router
1301                             Uuid networkUuid = original.getNetworkId();
1302                             LOG.info("update : SNAT disabled for Router {}", routerName);
1303                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1304                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1305                                     removeFlowInvTx);
1306                         } else {
1307                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1308                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1309                         }
1310                     }
1311                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1312                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1313                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1314                     }
1315
1316                     //Check if the Update is on External IPs
1317                     LOG.debug("update : Checking if this is update on External IPs");
1318                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1319                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1320
1321                     //Check if the External IPs are added during the update.
1322                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1323                     addedExternalIps.removeAll(originalExternalIps);
1324                     if (addedExternalIps.size() != 0) {
1325                         LOG.debug("update : Start processing of the External IPs addition during the update "
1326                                 + "operation");
1327                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1328                                 update.getExtGwMacAddress(), dpnId,
1329                                 update.getNetworkId(), null);
1330
1331                         for (String addedExternalIp : addedExternalIps) {
1332                 /*
1333                     1) Do nothing in the IntExtIp model.
1334                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1335                  */
1336                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1337                             String externalIp = externalIpParts[0];
1338                             String externalIpPrefix = externalIpParts[1];
1339                             String externalpStr = externalIp + "/" + externalIpPrefix;
1340                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1341                                             + "router ID {} in the ExternalIpsCounter model.",
1342                                     externalpStr, routerId);
1343                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1344                         }
1345                         LOG.debug(
1346                                 "update : End processing of the External IPs addition during the update operation");
1347                     }
1348
1349                     //Check if the External IPs are removed during the update.
1350                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1351                     removedExternalIps.removeAll(updatedExternalIps);
1352                     if (removedExternalIps.size() > 0) {
1353                         LOG.debug("update : Start processing of the External IPs removal during the update "
1354                                 + "operation");
1355                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1356                                 removedExternalIps, original.getExtGwMacAddress(),
1357                                 dpnId, networkId);
1358
1359                         for (String removedExternalIp : removedExternalIps) {
1360                 /*
1361                     1) Remove the mappings in the IntExt IP model which has external IP.
1362                     2) Remove the external IP in the ExternalCounter model.
1363                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1364                        least loaded external IP.
1365                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1366                     4) Increase the count of the allocated external IP by one.
1367                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1368                      i.e. the route for the external IP is absent.
1369                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1370                      the removed external IPs and also from the model.
1371                     7) Advertise to the BGP for removing the route for the removed external IPs.
1372                  */
1373
1374                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1375                             String externalIp = externalIpParts[0];
1376                             String externalIpPrefix = externalIpParts[1];
1377                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1378
1379                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1380                                     + "entries for removed external IP {}", externalIpAddrStr);
1381                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1382                             String vpnName = "";
1383                             if (vpnUuId != null) {
1384                                 vpnName = vpnUuId.getValue();
1385                             }
1386                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1387                                     update.getExtGwMacAddress(), removeFlowInvTx);
1388
1389                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1390                             //Get the internal IPs which are associated to the removed external IPs
1391                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1392                             List<String> removedInternalIps = new ArrayList<>();
1393                             for (IpMap ipMap : ipMaps) {
1394                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1395                                     removedInternalIps.add(ipMap.getInternalIp());
1396                                 }
1397                             }
1398
1399                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1400                             for (String removedInternalIp : removedInternalIps) {
1401                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1402                                                 + "router ID {} from the IntExtIP model",
1403                                         removedInternalIp, routerId);
1404                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1405                             }
1406
1407                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1408                                             + "router ID {} from the ExternalIpsCounter model.",
1409                                     externalIpAddrStr, routerId);
1410                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1411
1412                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1413                                     + "whose external IPs were removed.");
1414                             for (String removedInternalIp : removedInternalIps) {
1415                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1416                                         removedInternalIp, writeFlowInvTx);
1417                             }
1418
1419                             LOG.debug("update : Remove the NAPT translation entries from "
1420                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1421                             //Get the internalIP and internal Port which were associated to the removed external IP.
1422                             List<Integer> externalPorts = new ArrayList<>();
1423                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1424                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1425                                     .builder(IntextIpPortMap.class)
1426                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1427                             Optional<IpPortMapping> ipPortMapping =
1428                                     MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1429                             if (ipPortMapping.isPresent()) {
1430                                 List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1431                                         .getIntextIpProtocolType();
1432                                 for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1433                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1434                                     List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1435                                     for (IpPortMap ipPortMap : ipPortMaps) {
1436                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1437                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1438                                             externalPorts.add(ipPortExternal.getPortNum());
1439                                             List<String> removedInternalIpPorts =
1440                                                     protoTypesIntIpPortsMap.get(protoType);
1441                                             if (removedInternalIpPorts != null) {
1442                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1443                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1444                                             } else {
1445                                                 removedInternalIpPorts = new ArrayList<>();
1446                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1447                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1448                                             }
1449                                         }
1450                                     }
1451                                 }
1452                             }
1453
1454                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1455                             // the removed external IP.
1456                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1457                                     protoTypesIntIpPortsMap.entrySet();
1458                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1459                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1460                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1461                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1462                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1463                                     // Remove the IP port map from the intext-ip-port-map model,
1464                                     // which were containing the removed external IP
1465                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1466                                             protocolType);
1467                                     //Remove the IP port incomint packer map.
1468                                     naptPacketInHandler.removeIncomingPacketMap(
1469                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1470                                     String[] removedInternalIpPortParts = removedInternalIpPort
1471                                             .split(NatConstants.COLON_SEPARATOR);
1472                                     if (removedInternalIpPortParts.length == 2) {
1473                                         String removedInternalIp = removedInternalIpPortParts[0];
1474                                         String removedInternalPort = removedInternalIpPortParts[1];
1475                                         List<String> removedInternalPortsList =
1476                                                 internalIpPortMap.get(removedInternalPort);
1477                                         if (removedInternalPortsList != null) {
1478                                             removedInternalPortsList.add(removedInternalPort);
1479                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1480                                         } else {
1481                                             removedInternalPortsList = new ArrayList<>();
1482                                             removedInternalPortsList.add(removedInternalPort);
1483                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1484                                         }
1485                                     }
1486                                 }
1487                             }
1488
1489                             // Delete the entry from SnatIntIpPortMap DS
1490                             Set<String> internalIps = internalIpPortMap.keySet();
1491                             for (String internalIp : internalIps) {
1492                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1493                                         + "model SnatIntIpPortMap", internalIp);
1494                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1495                             }
1496
1497                             naptManager.removeNaptPortPool(externalIp);
1498
1499                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1500                                     + "the removed external IP {}", externalIp);
1501                             for (Integer externalPort : externalPorts) {
1502                                 //Remove the NAPT translation entries from Inbound NAPT table
1503                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1504                                         routerId, externalIp, externalPort);
1505                             }
1506
1507                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1508                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1509                                 String internalIp = internalIpPort.getKey();
1510                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1511                                         + "for the removed internal IP {}", internalIp);
1512                                 List<String> internalPorts = internalIpPort.getValue();
1513                                 for (String internalPort : internalPorts) {
1514                                     //Remove the NAPT translation entries from Outbound NAPT table
1515                                     naptPacketInHandler.removeIncomingPacketMap(
1516                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1517                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1518                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1519                                             routerId, internalIp, Integer.parseInt(internalPort));
1520                                 }
1521                             }
1522                         }
1523                         LOG.debug(
1524                                 "update : End processing of the External IPs removal during the update operation");
1525                     }
1526
1527                     //Check if its Update on subnets
1528                     LOG.debug("update : Checking if this is update on subnets");
1529                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1530                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1531                     Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1532                     addedSubnetIds.removeAll(originalSubnetIds);
1533
1534                     //Check if the Subnet IDs are added during the update.
1535                     if (addedSubnetIds.size() != 0) {
1536                         LOG.debug(
1537                                 "update : Start processing of the Subnet IDs addition during the update operation");
1538                         for (Uuid addedSubnetId : addedSubnetIds) {
1539                 /*
1540                     1) Select the least loaded external IP for the subnet and store the mapping of the
1541                     subnet IP and the external IP in the IntExtIp model.
1542                     2) Increase the count of the selected external IP by one.
1543                     3) Advertise to the BGP if external IP is allocated for the first time for the
1544                     router i.e. the route for the external IP is absent.
1545                  */
1546                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1547                             if (subnetIp != null) {
1548                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1549                                         writeFlowInvTx);
1550                             }
1551                         }
1552                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1553                     }
1554
1555                     //Check if the Subnet IDs are removed during the update.
1556                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1557                     removedSubnetIds.removeAll(updatedSubnetIds);
1558                     if (removedSubnetIds.size() != 0) {
1559                         LOG.debug(
1560                                 "update : Start processing of the Subnet IDs removal during the update operation");
1561                         for (Uuid removedSubnetId : removedSubnetIds) {
1562                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1563                             if (subnetAddr != null) {
1564                     /*
1565                         1) Remove the subnet IP and the external IP in the IntExtIp map
1566                         2) Decrease the count of the coresponding external IP by one.
1567                         3) Advertise to the BGP for removing the routes of the corresponding external
1568                         IP if its not allocated to any other internal IP.
1569                     */
1570
1571                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1572                                         subnetAddr[0] + "/" + subnetAddr[1]);
1573                                 if (externalIp == null) {
1574                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1575                                             routerId, subnetAddr[0]);
1576                                     return;
1577                                 }
1578
1579                                 naptManager.updateCounter(routerId, externalIp, false);
1580                                 // Traverse entire model of external-ip counter whether external ip is not
1581                                 // used by any other internal ip in any router
1582                                 if (!isExternalIpAllocated(externalIp)) {
1583                                     LOG.debug("update : external ip is not allocated to any other "
1584                                             + "internal IP so proceeding to remove routes");
1585                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1586                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1587                                             removeFlowInvTx);
1588                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1589                                                     + "router {} with networkId {} and externalIp {}",
1590                                             dpnId, routerId, networkId, externalIp);
1591                                 }
1592
1593                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1594                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1595                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1596                             }
1597                         }
1598                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1599                     }
1600                 }));
1601             }));
1602             return futures;
1603         }, NatConstants.NAT_DJC_MAX_RETRIES);
1604     }
1605
1606     private boolean isExternalIpAllocated(String externalIp) {
1607         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1608         Optional<ExternalIpsCounter> externalCountersData =
1609             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1610         if (externalCountersData.isPresent()) {
1611             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1612             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1613             for (ExternalCounters ext : externalCounters) {
1614                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1615                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1616                         if (externalIpCount.getCounter() != 0) {
1617                             return true;
1618                         }
1619                         break;
1620                     }
1621                 }
1622             }
1623         }
1624         return false;
1625     }
1626
1627     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1628             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1629         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1630         try {
1631             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1632             if (address instanceof Inet6Address) {
1633                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1634                 return;
1635             }
1636         } catch (UnknownHostException e) {
1637             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1638             return;
1639         }
1640         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1641         if (leastLoadedExtIpAddr != null) {
1642             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1643             String leastLoadedExtIp = externalIpParts[0];
1644             String leastLoadedExtIpPrefix = externalIpParts[1];
1645             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1646             subnetIp = subnetIpParts[0];
1647             String subnetIpPrefix = subnetIpParts[1];
1648             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1649             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1650                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1651                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1652             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1653
1654
1655             // Check if external IP is already assigned a route. (i.e. External IP is previously
1656             // allocated to any of the subnets)
1657             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1658             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1659             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1660             if (label != null) {
1661                 //update
1662                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1663                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1664                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1665                     label, internalIp, leastLoadedExtIpAddrStr);
1666                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1667                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1668                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1669                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1670                 return;
1671             }
1672
1673             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1674             // for the first time and hence not having a route.
1675             //Get the VPN Name using the network ID
1676             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1677             if (vpnName != null) {
1678                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1679                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1680                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1681                         + "added after gateway-set");
1682                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1683                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1684                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1685                         return;
1686                     }
1687                 }
1688                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1689                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1690                     writeFlowInvTx);
1691             }
1692         }
1693     }
1694
1695     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1696         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1697         for (IpMap ipMap : ipMaps) {
1698             if (ipMap.getExternalIp().equals(externalIp)) {
1699                 if (ipMap.getLabel() != null) {
1700                     return ipMap.getLabel();
1701                 }
1702             }
1703         }
1704         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1705         return null;
1706     }
1707
1708     @Override
1709     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1710         LOG.trace("remove : Router delete method");
1711         /*
1712         ROUTER DELETE SCENARIO
1713         1) Get the router ID from the event.
1714         2) Build the cookie information from the router ID.
1715         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1716         4) Build the flow with the cookie value.
1717         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1718         6) Remove the flows from the other switches which points to the primary and secondary
1719          switches for the flows related the router ID.
1720         7) Get the list of external IP address maintained for the router ID.
1721         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1722         9) Withdraw the corresponding routes from the BGP.
1723          */
1724
1725         if (identifier == null || router == null) {
1726             LOG.error("remove : returning without processing since routers is null");
1727             return;
1728         }
1729
1730         String routerName = router.getRouterName();
1731         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1732             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1733                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1734                         routerName);
1735                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1736                 if (routerId == NatConstants.INVALID_ID) {
1737                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1738                             routerName);
1739                     return;
1740                 }
1741                 long bgpVpnId = NatConstants.INVALID_ID;
1742                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1743                 if (bgpVpnUuid != null) {
1744                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1745                 }
1746                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1747                         tx);
1748                 Uuid networkUuid = router.getNetworkId();
1749
1750                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1751                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1752                     // No NAPT switch for external router, probably because the router is not attached to
1753                     // any
1754                     // internal networks
1755                     LOG.debug(
1756                             "No NAPT switch for router {}, check if router is attached to any internal "
1757                                     + "network",
1758                             routerName);
1759                     return;
1760                 } else {
1761                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1762                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1763                             routerId, tx);
1764                 }
1765                 NatOverVxlanUtil.releaseVNI(routerName, idManager);
1766             })), NatConstants.NAT_DJC_MAX_RETRIES);
1767     }
1768
1769     // TODO Clean up the exception handling
1770     @SuppressWarnings("checkstyle:IllegalCatch")
1771     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1772                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1773                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1774         LOG.info("handleDisableSnat : Entry");
1775         String routerName = router.getRouterName();
1776         try {
1777             if (routerFlag) {
1778                 removeNaptSwitch(routerName);
1779             } else {
1780                 updateNaptSwitch(routerName, BigInteger.ZERO);
1781             }
1782
1783             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1784             naptManager.removeExternalCounter(routerId);
1785
1786             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1787             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1788                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1789                     + "router ID {} from RouterNaptSwitch model", routerId);
1790                 return;
1791             }
1792             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1793                     networkUuid);
1794             if (extNwProvType == null) {
1795                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1796                 return;
1797             }
1798             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1799             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1800                     externalSubnetList, removeFlowInvTx, extNwProvType);
1801             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1802             try {
1803                 String externalSubnetVpn = null;
1804                 for (Uuid externalSubnetId : externalSubnetList) {
1805                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1806                     // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1807                     if (externalSubnet.isPresent()) {
1808                         externalSubnetVpn =  externalSubnetId.getValue();
1809                         clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1810                                 router.getExtGwMacAddress(), removeFlowInvTx);
1811                     }
1812                 }
1813                 if (externalSubnetVpn == null) {
1814                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1815                             router.getExtGwMacAddress(), removeFlowInvTx);
1816                 }
1817             } catch (Exception ex) {
1818                 LOG.error("handleDisableSnat : Failed to remove fib entries for routerId {} in naptSwitchDpnId {}",
1819                     routerId, naptSwitchDpnId, ex);
1820             }
1821             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1822             // for the router ID.
1823             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1824                     + "router ID {} in the DS", routerId);
1825             naptManager.removeMapping(routerId);
1826         } catch (Exception ex) {
1827             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, ex);
1828         }
1829         LOG.info("handleDisableSnat : Exit");
1830     }
1831
1832     // TODO Clean up the exception handling
1833     @SuppressWarnings("checkstyle:IllegalCatch")
1834     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1835                                              @Nonnull Collection<String> externalIps,
1836                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1837         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1838                 + "with internet vpn {}", routerName, vpnId);
1839         try {
1840             BigInteger naptSwitchDpnId = null;
1841             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1842                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1843             Optional<RouterToNaptSwitch> rtrToNapt =
1844                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1845             if (rtrToNapt.isPresent()) {
1846                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1847             }
1848             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1849
1850             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1851                     writeFlowInvTx);
1852             try {
1853                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1854                 if (extGwMacAddress != null) {
1855                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1856                             + "External Router ID {}", extGwMacAddress, routerId);
1857                 } else {
1858                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1859                             + "External Router ID {}", routerId);
1860                     return;
1861                 }
1862                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1863                         writeFlowInvTx);
1864             } catch (Exception ex) {
1865                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1866                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1867             }
1868             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1869         } catch (Exception ex) {
1870             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1871                     + "with internet vpn {}", routerName, vpnId, ex);
1872         }
1873         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1874                 routerName, vpnId);
1875     }
1876
1877     // TODO Clean up the exception handling
1878     @SuppressWarnings("checkstyle:IllegalCatch")
1879     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1880         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1881             .setPrimarySwitchId(naptSwitchId).build();
1882         try {
1883             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1884                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1885         } catch (Exception ex) {
1886             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1887                 naptSwitchId, routerName);
1888         }
1889         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1890             naptSwitchId, routerName);
1891     }
1892
1893     protected void removeNaptSwitch(String routerName) {
1894         // Remove router and switch from model
1895         InstanceIdentifier<RouterToNaptSwitch> id =
1896             InstanceIdentifier.builder(NaptSwitches.class)
1897                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1898         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1899         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1900         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1901         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1902     }
1903
1904     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1905                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1906                                                 @Nonnull Collection<String> externalIps,
1907                                                 Collection<Uuid> externalSubnetList,
1908                                                 TypedReadWriteTransaction<Configuration> confTx,
1909                                                 ProviderTypes extNwProvType) {
1910         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1911         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1912
1913         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1914         // traffic which comes from the  VMs of the NAPT switches)
1915         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1916         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1917
1918         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1919                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1920         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1921
1922         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1923         // traffic which comes from the VMs of the non NAPT switches)
1924         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1925                 routerName);
1926         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1927         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1928         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1929                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1930         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1931
1932         //Remove the flow table 25->44 from NAPT Switch
1933         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1934             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1935         }
1936
1937         //Remove the Outbound flow entry which forwards the packet to FIB Table
1938         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1939                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1940
1941         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1942                 NwConstants.IP_PROT_TCP);
1943         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1944             outboundTcpNatFlowRef);
1945         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1946
1947         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1948                 NwConstants.IP_PROT_UDP);
1949         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1950                 outboundUdpNatFlowRef);
1951         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1952
1953         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1954                 NwConstants.IP_PROT_ICMP);
1955         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1956                 icmpDropFlowRef);
1957         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1958
1959         removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1960         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1961         // External Subnet Vpn Id.
1962         for (Uuid externalSubnetId : externalSubnetList) {
1963             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1964             if (subnetVpnId != -1) {
1965                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1966                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1967                         natPfibSubnetFlowRef);
1968                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1969                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1970                         + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1971                         subnetVpnId, dpnId);
1972             }
1973         }
1974
1975         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1976         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1977         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1978
1979         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1980                 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1981         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1982
1983         // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1984         // - This does not work since ext-routers is deleted already - no network info
1985         //Get the VPN ID from the ExternalNetworks model
1986         long vpnId = -1;
1987         if (vpnName == null || vpnName.isEmpty()) {
1988             // ie called from router delete cases
1989             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1990             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1991             if (vpnUuid != null) {
1992                 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1993                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete or "
1994                         + "disableSNAT scenario", vpnId, networkId);
1995             }
1996         } else {
1997             // ie called from disassociate vpn case
1998             LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
1999                     vpnName);
2000             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2001             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}", vpnId);
2002         }
2003
2004         if (vpnId != NatConstants.INVALID_ID) {
2005             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2006             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2007             FlowEntity natPfibVpnFlowEntity =
2008                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2009             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the DPN ID {} "
2010                     + "and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2011             mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2012         }
2013
2014         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2015         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2016         if (ipPortMapping == null) {
2017             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2018             return;
2019         }
2020
2021         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2022         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2023             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2024             for (IpPortMap ipPortMap : ipPortMaps) {
2025                 String ipPortInternal = ipPortMap.getIpPortInternal();
2026                 String[] ipPortParts = ipPortInternal.split(":");
2027                 if (ipPortParts.length != 2) {
2028                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2029                     return;
2030                 }
2031                 String internalIp = ipPortParts[0];
2032                 String internalPort = ipPortParts[1];
2033
2034                 //Build the flow for the outbound NAPT table
2035                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2036                         + NatConstants.COLON_SEPARATOR + internalPort);
2037                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2038                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2039                 FlowEntity outboundNaptFlowEntity =
2040                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2041
2042                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2043                         + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2044                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2045
2046                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2047                 String externalIp = ipPortExternal.getIpAddress();
2048                 int externalPort = ipPortExternal.getPortNum();
2049
2050                 //Build the flow for the inbound NAPT table
2051                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2052                     String.valueOf(routerId), externalIp, externalPort);
2053                 FlowEntity inboundNaptFlowEntity =
2054                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2055
2056                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2057                         + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2058                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2059             }
2060         }
2061     }
2062
2063     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2064                                                     @Nonnull Collection<String> externalIps,
2065                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2066         long extVpnId = NatConstants.INVALID_ID;
2067         if (networkId != null) {
2068             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2069             if (vpnUuid != null) {
2070                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2071             } else {
2072                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2073             }
2074         } else {
2075             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2076             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2077         }
2078         if (extVpnId == NatConstants.INVALID_ID) {
2079             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2080             extVpnId = routerId;
2081         }
2082         for (String ip : externalIps) {
2083             String extIp = removeMaskFromIp(ip);
2084             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2085             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2086                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2087                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2088             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2089             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2090         }
2091     }
2092
2093     private String removeMaskFromIp(String ip) {
2094         if (ip != null && !ip.trim().isEmpty()) {
2095             return ip.split("/")[0];
2096         }
2097         return ip;
2098     }
2099
2100     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2101                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2102                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2103         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2104         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2105
2106         //Remove the NAPT PFIB TABLE entry
2107         long vpnId = -1;
2108         if (vpnName != null) {
2109             // ie called from disassociate vpn case
2110             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2111                     + "with vpnName {}", vpnName);
2112             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2113             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2114                     vpnId);
2115         }
2116
2117         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2118                 networkId, routerName, dpnId)) {
2119             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2120             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2121             FlowEntity natPfibVpnFlowEntity =
2122                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2123             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2124                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2125             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2126
2127             // Remove IP-PORT active NAPT entries and release port from IdManager
2128             // For the router ID get the internal IP , internal port and the corresponding
2129             // external IP and external Port.
2130             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2131             if (ipPortMapping == null) {
2132                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2133                 return;
2134             }
2135             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2136             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2137                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2138                 for (IpPortMap ipPortMap : ipPortMaps) {
2139                     String ipPortInternal = ipPortMap.getIpPortInternal();
2140                     String[] ipPortParts = ipPortInternal.split(":");
2141                     if (ipPortParts.length != 2) {
2142                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2143                                 + "and port");
2144                         return;
2145                     }
2146                     String internalIp = ipPortParts[0];
2147                     String internalPort = ipPortParts[1];
2148
2149                     //Build the flow for the outbound NAPT table
2150                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2151                             + NatConstants.COLON_SEPARATOR + internalPort);
2152                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2153                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2154                     FlowEntity outboundNaptFlowEntity =
2155                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2156
2157                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2158                             + "active switch with the DPN ID {} and router ID {}",
2159                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2160                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2161
2162                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2163                     String externalIp = ipPortExternal.getIpAddress();
2164                     int externalPort = ipPortExternal.getPortNum();
2165
2166                     //Build the flow for the inbound NAPT table
2167                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2168                         String.valueOf(routerId), externalIp, externalPort);
2169                     FlowEntity inboundNaptFlowEntity =
2170                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2171
2172                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2173                             + "active active switch with the DPN ID {} and router ID {}",
2174                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2175                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2176
2177                     // Finally release port from idmanager
2178                     String internalIpPort = internalIp + ":" + internalPort;
2179                     naptManager.removePortFromPool(internalIpPort, externalIp);
2180
2181                     //Remove sessions from models
2182                     naptManager.removeIpPortMappingForRouterID(routerId);
2183                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2184                 }
2185             }
2186         } else {
2187             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2188         }
2189     }
2190
2191     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2192             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2193         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2194
2195         // Remove the flows from the other switches which points to the primary and secondary switches
2196         // for the flows related the router ID.
2197         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2198         if (allSwitchList.isEmpty()) {
2199             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2200             return;
2201         }
2202         for (BigInteger dpnId : allSwitchList) {
2203             if (!naptSwitchDpnId.equals(dpnId)) {
2204                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2205
2206                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2207                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2208                 FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2209
2210                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2211                         + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2212                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2213
2214                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2215                 long groupId = createGroupId(getGroupIdKey(routerName));
2216                 List<BucketInfo> listBucketInfo = new ArrayList<>();
2217                 GroupEntity preSnatGroupEntity =
2218                     MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, listBucketInfo);
2219
2220                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2221                         + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2222                 mdsalManager.removeGroup(removeFlowInvTx, preSnatGroupEntity);
2223
2224             }
2225         }
2226     }
2227
2228     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2229                                          @Nonnull Collection<String> externalIps, String vpnName,
2230                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx) {
2231         //Withdraw the corresponding routes from the BGP.
2232         //Get the network ID using the router ID.
2233         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2234                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2235         if (networkUuid == null) {
2236             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2237             return;
2238         }
2239
2240         if (externalIps.isEmpty()) {
2241             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2242             return;
2243         }
2244
2245         if (vpnName == null) {
2246             //Get the VPN Name using the network ID
2247             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2248             if (vpnName == null) {
2249                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2250                     networkUuid, routerId);
2251                 return;
2252             }
2253         }
2254         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2255
2256         //Remove custom FIB routes
2257         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2258         for (String extIp : externalIps) {
2259             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2260         }
2261     }
2262
2263     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2264                                             final Uuid networkUuid, String extGwMacAddress,
2265                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2266         clearBgpRoutes(extIp, vpnName);
2267         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2268                 removeFlowInvTx);
2269     }
2270
2271     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2272                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2273                                              String gwMacAddress, boolean switchOver,
2274                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2275         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2276         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2277         if (routerName == null) {
2278             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2279             return;
2280         }
2281         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2282         if (extNwProvType == null) {
2283             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2284             return;
2285         }
2286         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2287          * external network provided type is VxLAN
2288          */
2289         if (extNwProvType == ProviderTypes.VXLAN) {
2290             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2291             );
2292             return;
2293         }
2294         if (tempLabel < 0) {
2295             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2296                     extIp, routerId);
2297             return;
2298         }
2299
2300         final long label = tempLabel;
2301         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2302         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2303                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2304                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2305         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2306
2307         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2308         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2309         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2310             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2311             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2312         }
2313         if (!switchOver) {
2314             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2315                 Futures.transformAsync(future, result -> {
2316                     //Release label
2317                     if (result.isSuccessful()) {
2318                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2319                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2320                             .setVpnName(vpnName).setIpPrefix(externalIp).build();
2321                         return vpnService.removeVpnLabel(labelInput);
2322                     } else {
2323                         String errMsg =
2324                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2325                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2326                         LOG.error(errMsg);
2327                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2328                     }
2329                 }, MoreExecutors.directExecutor());
2330
2331             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2332
2333                 @Override
2334                 public void onFailure(@Nonnull Throwable error) {
2335                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2336                         + "got external ip {}", label, extIp, error);
2337                 }
2338
2339                 @Override
2340                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2341                     if (result.isSuccessful()) {
2342                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2343                             + "from VPN {}", externalIp, vpnName);
2344                     } else {
2345                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2346                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2347                     }
2348                 }
2349             }, MoreExecutors.directExecutor());
2350         } else {
2351             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2352                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2353         }
2354     }
2355
2356     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2357                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2358                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
2359         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2360         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2361         if (routerName == null) {
2362             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2363             return;
2364         }
2365         //Get the external network provider type from networkId
2366         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2367         if (extNwProvType == null) {
2368             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2369                     networkUuid);
2370             return;
2371         }
2372         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2373          *  external network provided type is VxLAN
2374          */
2375         if (extNwProvType == ProviderTypes.VXLAN) {
2376             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2377             return;
2378         }
2379         //Get IPMaps from the DB for the router ID
2380         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2381         if (dbIpMaps.isEmpty()) {
2382             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2383             return;
2384         }
2385
2386         long tempLabel = NatConstants.INVALID_ID;
2387         for (IpMap dbIpMap : dbIpMaps) {
2388             String dbExternalIp = dbIpMap.getExternalIp();
2389             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2390             //Select the IPMap, whose external IP is the IP for which FIB is installed
2391             if (extIp.equals(dbExternalIp)) {
2392                 tempLabel = dbIpMap.getLabel();
2393                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2394                     tempLabel, dbExternalIp, routerId);
2395                 break;
2396             }
2397         }
2398         if (tempLabel == NatConstants.INVALID_ID) {
2399             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2400                     extIp, routerId);
2401             return;
2402         }
2403
2404         final long label = tempLabel;
2405         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2406         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2407                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2408                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2409         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2410
2411         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2412         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2413         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2414             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2415             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2416         }
2417         if (!switchOver) {
2418             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2419                     Futures.transformAsync(future, result -> {
2420                         //Release label
2421                         if (result.isSuccessful()) {
2422                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2423                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2424                             return vpnService.removeVpnLabel(labelInput);
2425                         } else {
2426                             String errMsg =
2427                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2428                                             + "prefix %s Failed - %s",
2429                                             dpnId, externalIp, result.getErrors());
2430                             LOG.error(errMsg);
2431                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2432                         }
2433                     }, MoreExecutors.directExecutor());
2434
2435             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2436
2437                 @Override
2438                 public void onFailure(@Nonnull Throwable error) {
2439                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2440                 }
2441
2442                 @Override
2443                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2444                     if (result.isSuccessful()) {
2445                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2446                             + "from VPN {}", externalIp, vpnName);
2447                     } else {
2448                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2449                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2450                     }
2451                 }
2452             }, MoreExecutors.directExecutor());
2453         } else {
2454             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2455                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2456         }
2457     }
2458
2459     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2460                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2461                                                TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2462         //Withdraw the corresponding routes from the BGP.
2463         //Get the network ID using the router ID.
2464         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2465                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2466         if (networkUuid == null) {
2467             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2468             return;
2469         }
2470
2471         if (externalIps == null || externalIps.isEmpty()) {
2472             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2473             return;
2474         }
2475
2476         if (vpnName == null) {
2477             //Get the VPN Name using the network ID
2478             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2479             if (vpnName == null) {
2480                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2481                     networkUuid, routerId);
2482                 return;
2483             }
2484         }
2485         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2486
2487         //Remove custom FIB routes
2488         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2489         for (String extIp : externalIps) {
2490             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2491                     writeFlowInvTx);
2492         }
2493     }
2494
2495     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2496         //Inform BGP about the route removal
2497         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2498         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2499         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2500     }
2501
2502     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2503         TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2504         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2505         List<MatchInfo> mkMatches = new ArrayList<>();
2506         // Matching metadata
2507         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
2508         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
2509             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
2510             5, String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
2511             COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
2512         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowEntity);
2513         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2514     }
2515
2516     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2517         TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
2518         List<MatchInfo> matches = new ArrayList<>();
2519         matches.add(MatchEthernetType.MPLS_UNICAST);
2520         matches.add(new MatchMplsLabel(serviceId));
2521
2522         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2523
2524         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2525
2526         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
2527             10, flowRef, 0, 0,
2528             COOKIE_VM_LFIB_TABLE, matches, null);
2529
2530         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowEntity);
2531
2532         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2533     }
2534
2535     /**
2536      * router association to vpn.
2537      *
2538      * @param routerName - Name of router
2539      * @param routerId - router id
2540      * @param bgpVpnName BGP VPN name
2541      */
2542     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2543             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2544         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2545         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2546             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2547
2548             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2549
2550             if (bgpVpnId != NatConstants.INVALID_ID) {
2551                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2552                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2553                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2554                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2555                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2556                     getRoutersIdentifier(bgpVpnId), rtrs);
2557
2558                 // Get the allocated Primary NAPT Switch for this router
2559                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2560
2561                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2562                         routerId, bgpVpnId);
2563                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2564
2565                 // Get the group ID
2566                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2567                 createGroupId(getGroupIdKey(routerName));
2568                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2569                         extNwProvType);
2570             }
2571         }
2572     }
2573
2574     /**
2575      * router disassociation from vpn.
2576      *
2577      * @param routerName - Name of router
2578      * @param routerId - router id
2579      * @param bgpVpnName BGP VPN name
2580      */
2581     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2582             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2583         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2584         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2585             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2586             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2587
2588             // Get the allocated Primary NAPT Switch for this router
2589             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2590
2591             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2592             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2593
2594             // Get the group ID
2595             createGroupId(getGroupIdKey(routerName));
2596             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2597             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2598                     writeFlowInvTx, extNwProvType);
2599         }
2600     }
2601
2602     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2603         InstanceIdentifier<Routers> routerInstanceIndentifier =
2604             InstanceIdentifier.builder(ExtRouters.class)
2605                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2606         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2607         return routerData.isPresent() && routerData.get().isEnableSnat();
2608     }
2609
2610     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2611         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2612
2613         long changedVpnId = bgpVpnId;
2614         String idType = "BGP VPN";
2615         if (bgpVpnId == NatConstants.INVALID_ID) {
2616             changedVpnId = routerId;
2617             idType = "router";
2618         }
2619
2620         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2621         if (switches.isEmpty()) {
2622             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2623             return;
2624         }
2625         for (BigInteger dpnId : switches) {
2626             // Update the BGP VPN ID in the SNAT miss entry to group
2627             if (!dpnId.equals(primarySwitchId)) {
2628                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2629                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2630                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2631                 long groupId = createGroupId(getGroupIdKey(routerName));
2632                 if (!isSnatCfgd) {
2633                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2634                 }
2635
2636                 LOG.debug(
2637                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2638                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2639                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2640                 mdsalManager.addFlow(confTx, flowEntity);
2641             } else {
2642                 LOG.debug(
2643                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2644                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2645                 FlowEntity flowEntity =
2646                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2647                 mdsalManager.addFlow(confTx, flowEntity);
2648
2649                 LOG.debug(
2650                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2651                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2652                         idType, changedVpnId, primarySwitchId);
2653                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2654                         changedVpnId, confTx, extNwProvType);
2655
2656                 LOG.debug(
2657                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2658                                 + "which punts the packet to the controller in the Primary switch {}",
2659                         idType, changedVpnId, primarySwitchId);
2660                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2661
2662                 LOG.debug(
2663                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2664                                 + " outgoing packet to FIB Table in the Primary switch {}",
2665                         idType, changedVpnId, primarySwitchId);
2666                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2667
2668                 LOG.debug(
2669                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2670                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2671                                 + " {}", idType, changedVpnId, primarySwitchId);
2672                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2673
2674                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2675                         primarySwitchId);
2676                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2677                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2678                 if (vpnId != NatConstants.INVALID_ID) {
2679                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2680                 }
2681             }
2682         }
2683     }
2684
2685     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2686         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2687         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2688         if (ipPortMapping == null) {
2689             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2690             return;
2691         }
2692         // Get the External Gateway MAC Address
2693         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2694         if (extGwMacAddress != null) {
2695             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2696                     extGwMacAddress, routerId);
2697         } else {
2698             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2699                     routerId);
2700             return;
2701         }
2702         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2703         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2704             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2705             for (IpPortMap ipPortMap : ipPortMaps) {
2706                 String ipPortInternal = ipPortMap.getIpPortInternal();
2707                 String[] ipPortParts = ipPortInternal.split(":");
2708                 if (ipPortParts.length != 2) {
2709                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2710                     return;
2711                 }
2712                 String internalIp = ipPortParts[0];
2713                 String internalPort = ipPortParts[1];
2714                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2715                         internalIp, internalPort);
2716                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2717                 NAPTEntryEvent.Protocol protocol;
2718                 switch (protocolTypes) {
2719                     case TCP:
2720                         protocol = NAPTEntryEvent.Protocol.TCP;
2721                         break;
2722                     case UDP:
2723                         protocol = NAPTEntryEvent.Protocol.UDP;
2724                         break;
2725                     default:
2726                         protocol = NAPTEntryEvent.Protocol.TCP;
2727                 }
2728                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2729                 SessionAddress externalAddress =
2730                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2731                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2732                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2733                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2734                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2735                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2736             }
2737         }
2738     }
2739
2740     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2741                                                           long changedVpnId) {
2742
2743         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2744             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2745         List<MatchInfo> matches = new ArrayList<>();
2746         matches.add(MatchEthernetType.IPV4);
2747         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2748
2749         List<ActionInfo> actionsInfo = new ArrayList<>();
2750         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2751                 routerName);
2752         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2753         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2754                 actionsInfo);
2755         actionsInfo.add(new ActionGroup(groupId));
2756         List<InstructionInfo> instructions = new ArrayList<>();
2757         instructions.add(new InstructionApplyActions(actionsInfo));
2758         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2759         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2760             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2761             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2762
2763         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2764         return flowEntity;
2765     }
2766
2767     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2768                                                                         long changedVpnId) {
2769
2770         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2771                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2772         List<MatchInfo> matches = new ArrayList<>();
2773         matches.add(MatchEthernetType.IPV4);
2774         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2775
2776         List<InstructionInfo> instructions = new ArrayList<>();
2777         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2778
2779         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2780         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2781             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2782             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2783
2784         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2785         return flowEntity;
2786     }
2787
2788     // TODO : Replace this with ITM Rpc once its available with full functionality
2789     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2790         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2791
2792         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2793             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2794         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2795                 extNwProvType);
2796         mdsalManager.addFlow(confTx, flowEntity);
2797     }
2798
2799     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2800             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2801         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2802             dpId, routerName, changedVpnId);
2803         List<MatchInfo> matches = new ArrayList<>();
2804         matches.add(MatchEthernetType.IPV4);
2805
2806         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2807         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2808             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2809         }
2810         matches.add(new MatchTunnelId(tunnelId));
2811
2812         List<InstructionInfo> instructions = new ArrayList<>();
2813         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2814             MetaDataUtil.METADATA_MASK_VRFID));
2815         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2816         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2817         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2818         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2819             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2820             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2821         return flowEntity;
2822     }
2823
2824     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2825                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2826         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2827             dpnId, routerId, changedVpnId);
2828         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2829                 NwConstants.IP_PROT_TCP);
2830         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2831         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2832
2833         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2834                 NwConstants.IP_PROT_UDP);
2835         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2836         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2837
2838         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2839         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2840         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2841     }
2842
2843     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2844                                                            long changedVpnId, int protocol) {
2845         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2846             dpId, routerId, changedVpnId);
2847         BigInteger cookie = getCookieOutboundFlow(routerId);
2848         List<MatchInfo> matches = new ArrayList<>();
2849         matches.add(MatchEthernetType.IPV4);
2850         matches.add(new MatchIpProtocol((short)protocol));
2851         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2852
2853         List<InstructionInfo> instructions = new ArrayList<>();
2854         List<ActionInfo> actionsInfos = new ArrayList<>();
2855         actionsInfos.add(new ActionPuntToController());
2856         if (snatPuntTimeout != 0) {
2857             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2858         }
2859         instructions.add(new InstructionApplyActions(actionsInfos));
2860
2861         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2862         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2863             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2864         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2865         return flowEntity;
2866     }
2867
2868     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2869                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2870         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2871             dpnId, segmentId, changedVpnId);
2872         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2873         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2874     }
2875
2876     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2877
2878         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2879             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2880         List<MatchInfo> matches = new ArrayList<>();
2881         matches.add(MatchEthernetType.IPV4);
2882         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2883
2884         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2885         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2886         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2887         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2888         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2889
2890         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2891         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2892             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2893             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2894         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2895         return flowEntity;
2896     }
2897
2898     @Override
2899     protected ExternalRoutersListener getDataTreeChangeListener() {
2900         return ExternalRoutersListener.this;
2901     }
2902
2903     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2904                                                             TypedWriteTransaction<Configuration> writeFlowInvTx) {
2905         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2906                 routerName);
2907         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2908             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2909             if (subnetVpnId != -1) {
2910                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2911                     + "and vpnId {}", dpnId, subnetVpnId);
2912                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2913             }
2914         }
2915     }
2916 }