natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.FutureCallback;
  12 import com.google.common.util.concurrent.Futures;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14 import com.google.common.util.concurrent.MoreExecutors;
  15 import java.math.BigInteger;
  16 import java.net.Inet6Address;
  17 import java.net.InetAddress;
  18 import java.net.UnknownHostException;
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collection;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Set;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.Future;
  31 import javax.annotation.Nonnull;
  32 import javax.annotation.PostConstruct;
  33 import javax.inject.Inject;
  34 import javax.inject.Singleton;
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  37 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.BucketInfo;
  45 import org.opendaylight.genius.mdsalutil.FlowEntity;
  46 import org.opendaylight.genius.mdsalutil.GroupEntity;
  47 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.MatchInfo;
  50 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  51 import org.opendaylight.genius.mdsalutil.NwConstants;
  52 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  53 import org.opendaylight.genius.mdsalutil.UpgradeState;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  71 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  72 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  75 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  76 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  77 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  78 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  79 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  80 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  81 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  82 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  83 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  84 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 145 import org.opendaylight.yangtools.yang.binding.DataObject;
 146 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 147 import org.opendaylight.yangtools.yang.common.RpcResult;
 148 import org.slf4j.Logger;
 149 import org.slf4j.LoggerFactory;
 150
 151 @Singleton
 152 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 153     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 154
 155     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 156     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 157
 158     private final DataBroker dataBroker;
 159     private final ManagedNewTransactionRunner txRunner;
 160     private final IMdsalApiManager mdsalManager;
 161     private final ItmRpcService itmManager;
 162     private final OdlInterfaceRpcService odlInterfaceRpcService;
 163     private final IdManagerService idManager;
 164     private final NaptManager naptManager;
 165     private final NAPTSwitchSelector naptSwitchSelector;
 166     private final IBgpManager bgpManager;
 167     private final VpnRpcService vpnService;
 168     private final FibRpcService fibService;
 169     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 170     private final NaptEventHandler naptEventHandler;
 171     private final NaptPacketInHandler naptPacketInHandler;
 172     private final IFibManager fibManager;
 173     private final IVpnManager vpnManager;
 174     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 175     private final CentralizedSwitchScheduler  centralizedSwitchScheduler;
 176     private final NatMode natMode;
 177     private final INeutronVpnManager nvpnManager;
 178     private final IElanService elanManager;
 179     private final JobCoordinator coordinator;
 180     private final UpgradeState upgradeState;
 181     private final IInterfaceManager interfaceManager;
 182     private final int snatPuntTimeout;
 183
 184     @Inject
 185     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 186                                    final ItmRpcService itmManager,
 187                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 188                                    final IdManagerService idManager,
 189                                    final NaptManager naptManager,
 190                                    final NAPTSwitchSelector naptSwitchSelector,
 191                                    final IBgpManager bgpManager,
 192                                    final VpnRpcService vpnService,
 193                                    final FibRpcService fibService,
 194                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 195                                    final NaptEventHandler naptEventHandler,
 196                                    final NaptPacketInHandler naptPacketInHandler,
 197                                    final IFibManager fibManager,
 198                                    final IVpnManager vpnManager,
 199                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 200                                    final INeutronVpnManager nvpnManager,
 201                                    final CentralizedSwitchScheduler centralizedSwitchScheduler,
 202                                    final NatserviceConfig config,
 203                                    final IElanService elanManager,
 204                                    final JobCoordinator coordinator,
 205                                    final UpgradeState upgradeState,
 206                                    final IInterfaceManager interfaceManager) {
 207         super(Routers.class, ExternalRoutersListener.class);
 208         this.dataBroker = dataBroker;
 209         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 210         this.mdsalManager = mdsalManager;
 211         this.itmManager = itmManager;
 212         this.odlInterfaceRpcService = odlInterfaceRpcService;
 213         this.idManager = idManager;
 214         this.naptManager = naptManager;
 215         this.naptSwitchSelector = naptSwitchSelector;
 216         this.bgpManager = bgpManager;
 217         this.vpnService = vpnService;
 218         this.fibService = fibService;
 219         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 220         this.naptEventHandler = naptEventHandler;
 221         this.naptPacketInHandler = naptPacketInHandler;
 222         this.fibManager = fibManager;
 223         this.vpnManager = vpnManager;
 224         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 225         this.nvpnManager = nvpnManager;
 226         this.elanManager = elanManager;
 227         this.centralizedSwitchScheduler = centralizedSwitchScheduler;
 228         this.coordinator = coordinator;
 229         this.upgradeState = upgradeState;
 230         this.interfaceManager = interfaceManager;
 231         if (config != null) {
 232             this.natMode = config.getNatMode();
 233             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 234         } else {
 235             this.natMode = NatMode.Controller;
 236             this.snatPuntTimeout = 0;
 237         }
 238     }
 239
 240     @Override
 241     @PostConstruct
 242     public void init() {
 243         LOG.info("{} init", getClass().getSimpleName());
 244         // This class handles ExternalRouters for Controller SNAT mode.
 245         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 246         if (natMode == NatMode.Controller) {
 247             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 248             NatUtil.createGroupIdPool(idManager);
 249         }
 250     }
 251
 252     @Override
 253     protected InstanceIdentifier<Routers> getWildCardPath() {
 254         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 255     }
 256
 257     @Override
 258     // TODO Clean up the exception handling
 259     @SuppressWarnings("checkstyle:IllegalCatch")
 260     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 261         // Populate the router-id-name container
 262         String routerName = routers.getRouterName();
 263         LOG.info("add : external router event for {}", routerName);
 264         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 265         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 266         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 267         try {
 268             coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 269                 () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 270                     LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 271                     long bgpVpnId = NatConstants.INVALID_ID;
 272                     if (bgpVpnUuid != null) {
 273                         bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 274                     }
 275                     addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, tx);
 276                     // Allocate Primary Napt Switch for this router
 277                     BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 278                     if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 279                         if (!routers.isEnableSnat()) {
 280                             LOG.info("add : SNAT is disabled for external router {} ", routerName);
 281                             /* If SNAT is disabled on ext-router though L3_FIB_TABLE(21) -> PSNAT_TABLE(26) flow
 282                              * is required for DNAT. Hence writeFlowInvTx object submit is required.
 283                              */
 284                             return;
 285                         }
 286                         handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, tx);
 287                     }
 288                 })), NatConstants.NAT_DJC_MAX_RETRIES);
 289         } catch (Exception ex) {
 290             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 291                     routerName, ex);
 292         }
 293     }
 294
 295     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 296                                  WriteTransaction writeFlowInvTx) {
 297         String routerName = routers.getRouterName();
 298         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 299
 300         naptManager.initialiseExternalCounter(routers, routerId);
 301         subnetRegisterMapping(routers, routerId);
 302
 303         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 304             primarySwitchId, routerName);
 305
 306         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 307                 routers.getNetworkId());
 308         if (extNwProvType == null) {
 309             LOG.error("handleEnableSnat : External Network Provider Type missing");
 310             return;
 311         }
 312
 313         if (bgpVpnId != NatConstants.INVALID_ID) {
 314             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, writeFlowInvTx,
 315                     extNwProvType);
 316         } else {
 317             // write metadata and punt
 318             installOutboundMissEntry(routerName, routerId, primarySwitchId, writeFlowInvTx);
 319             // Now install entries in SNAT tables to point to Primary for each router
 320             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 321             for (BigInteger dpnId : switches) {
 322                 // Handle switches and NAPT switches separately
 323                 if (!dpnId.equals(primarySwitchId)) {
 324                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 325                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 326                 } else {
 327                     LOG.debug("handleEnableSnat : Handle NAPT switch");
 328                     handlePrimaryNaptSwitch(dpnId, routerName, routerId, writeFlowInvTx);
 329                 }
 330             }
 331         }
 332
 333         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 334         if (externalIps.isEmpty()) {
 335             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 336             return;
 337         } else {
 338             for (String externalIpAddrPrefix : externalIps) {
 339                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 340                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 341                 handleSnatReverseTraffic(primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix,
 342                         writeFlowInvTx);
 343             }
 344         }
 345         LOG.debug("handleEnableSnat : Exit");
 346     }
 347
 348     private BigInteger getPrimaryNaptSwitch(String routerName) {
 349         // Allocate Primary Napt Switch for this router
 350         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 351         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 352             LOG.debug("handleEnableSnat : Primary NAPT switch with DPN ID {} is already elected for router {}",
 353                 primarySwitchId, routerName);
 354             return primarySwitchId;
 355         }
 356         // Validating and creating VNI pool during when NAPT switch is selected.
 357         // With Assumption this might be the first NAT service comes up.
 358         NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 359                 NatConstants.ODL_VNI_POOL_NAME);
 360         // Allocated an id from VNI pool for the Router.
 361         NatOverVxlanUtil.getRouterVni(idManager, routerName, NatConstants.INVALID_ID);
 362         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 363         LOG.debug("handleEnableSnat : Primary NAPT switch DPN ID {}", primarySwitchId);
 364
 365         return primarySwitchId;
 366     }
 367
 368     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 369                                                      WriteTransaction writeFlowInvTx) {
 370         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 371         if (extVpnId == NatConstants.INVALID_ID) {
 372             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 373             extVpnId = routerId;
 374         }
 375         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 376         if (externalIps.isEmpty()) {
 377             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 378                 dpnId, extVpnId);
 379             return;
 380         }
 381         for (String ip : externalIps) {
 382             Uuid subnetId = getSubnetIdForFixedIp(ip);
 383             if (subnetId != null) {
 384                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 385                 if (subnetVpnId != NatConstants.INVALID_ID) {
 386                     extVpnId = subnetVpnId;
 387                 }
 388                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 389                     dpnId, extVpnId, subnetId);
 390                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 391                 mdsalManager.addFlowToTx(postNaptFlowEntity, writeFlowInvTx);
 392             }
 393         }
 394     }
 395
 396     private Uuid getSubnetIdForFixedIp(String ip) {
 397         if (ip != null) {
 398             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 399             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 400             Uuid subnetId = NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 401             return subnetId;
 402         }
 403         LOG.error("getSubnetIdForFixedIp : ip is null");
 404         return null;
 405     }
 406
 407     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 408         List<Uuid> subnetList = null;
 409         List<String> externalIps = null;
 410         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 411         subnetList = routerEntry.getSubnetIds();
 412         externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 413         int counter = 0;
 414         int extIpCounter = externalIps.size();
 415         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 416                 counter, extIpCounter);
 417         for (Uuid subnet : subnetList) {
 418             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 419             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 420                 .builder(Subnetmaps.class)
 421                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 422                 .build();
 423             Optional<Subnetmap> sn = read(dataBroker, LogicalDatastoreType.CONFIGURATION, subnetmapId);
 424             if (sn.isPresent()) {
 425                 // subnets
 426                 Subnetmap subnetmapEntry = sn.get();
 427                 String subnetString = subnetmapEntry.getSubnetIp();
 428                 String[] subnetSplit = subnetString.split("/");
 429                 String subnetIp = subnetSplit[0];
 430                 try {
 431                     InetAddress address = InetAddress.getByName(subnetIp);
 432                     if (address instanceof Inet6Address) {
 433                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 434                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 435                         continue;
 436                     }
 437                 } catch (UnknownHostException e) {
 438                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 439                     return;
 440                 }
 441                 String subnetPrefix = "0";
 442                 if (subnetSplit.length == 2) {
 443                     subnetPrefix = subnetSplit[1];
 444                 }
 445                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 446                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 447                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 448                 //externalIps
 449                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 450                         counter, extIpCounter);
 451                 if (extIpCounter != 0) {
 452                     if (counter < extIpCounter) {
 453                         String[] ipSplit = externalIps.get(counter).split("/");
 454                         String externalIp = ipSplit[0];
 455                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 456                         if (ipSplit.length == 2) {
 457                             extPrefix = ipSplit[1];
 458                         }
 459                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 460                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 461                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 462                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 463                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 464                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 465                     } else {
 466                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 467                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 468                         String[] ipSplit = externalIps.get(counter).split("/");
 469                         String externalIp = ipSplit[0];
 470                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 471                         if (ipSplit.length == 2) {
 472                             extPrefix = ipSplit[1];
 473                         }
 474                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 475                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 476                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 477                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 478                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 479                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 480                             externalIp, extPrefix);
 481                     }
 482                 }
 483                 counter++;
 484                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 485             } else {
 486                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 487             }
 488         }
 489     }
 490
 491     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 492             Uuid bgpVpnUuid, boolean create, WriteTransaction writeFlowInvTx) {
 493         //Check if BGP VPN exists. If exists then invoke the new method.
 494         if (bgpVpnId != NatConstants.INVALID_ID) {
 495             if (bgpVpnUuid != null) {
 496                 String bgpVpnName = bgpVpnUuid.getValue();
 497                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 498                     bgpVpnId, bgpVpnName);
 499                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 500                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 501                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 502                     getRoutersIdentifier(bgpVpnId), rtrs);
 503             }
 504             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, create, writeFlowInvTx);
 505             return;
 506         }
 507
 508         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 509         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, writeFlowInvTx);
 510     }
 511
 512     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 513             WriteTransaction writeFlowInvTx) {
 514         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 515         if (switches.isEmpty()) {
 516             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 517             return;
 518         }
 519         if (routerId == NatConstants.INVALID_ID) {
 520             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 521                     + "default NAT route in FIB", routerName);
 522             return;
 523         }
 524         for (BigInteger dpnId : switches) {
 525             if (create) {
 526                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 527                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 528                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 529             } else {
 530                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 531                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 532                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 533             }
 534         }
 535     }
 536
 537     private void addOrDelDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 538             long bgpVpnId, boolean create,WriteTransaction writeFlowInvTx) {
 539         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 540         if (dpnIds.isEmpty()) {
 541             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 542                     + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 543             return;
 544         }
 545         for (BigInteger dpnId : dpnIds) {
 546             if (create) {
 547                 if (bgpVpnId != NatConstants.INVALID_ID) {
 548                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 549                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 550                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 551                 } else {
 552                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 553                             + "in dpn {} for the internal vpn", routerId, dpnId);
 554                     defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId,writeFlowInvTx);
 555                 }
 556             } else {
 557                 if (bgpVpnId != NatConstants.INVALID_ID) {
 558                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 559                             + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 560                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, writeFlowInvTx);
 561                 } else {
 562                     LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 563                             + "in dpn {} for the internal vpn", routerId, dpnId);
 564                     defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, writeFlowInvTx);
 565                 }
 566             }
 567         }
 568     }
 569
 570     // TODO Clean up the exception handling
 571     @SuppressWarnings("checkstyle:IllegalCatch")
 572     public static <T extends DataObject> Optional<T> read(DataBroker broker, LogicalDatastoreType datastoreType,
 573                                                           InstanceIdentifier<T> path) {
 574         ReadOnlyTransaction tx = broker.newReadOnlyTransaction();
 575
 576         try {
 577             return tx.read(datastoreType, path).get();
 578         } catch (Exception e) {
 579             throw new RuntimeException(e);
 580         }
 581     }
 582
 583     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 584                                             WriteTransaction writeFlowInvTx) {
 585         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 586         if (routerId != NatConstants.INVALID_ID) {
 587             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 588                     primarySwitchId, routerId);
 589             createOutboundTblEntry(primarySwitchId, routerId, writeFlowInvTx);
 590         } else {
 591             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 592                 + "createAndInstallMissEntry", routerName);
 593         }
 594     }
 595
 596     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 597         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 598                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 599     }
 600
 601     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 602         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 603                 .FLOWID_SEPARATOR + vpnId;
 604     }
 605
 606     public BigInteger getCookieOutboundFlow(long routerId) {
 607         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 608             BigInteger.valueOf(routerId));
 609     }
 610
 611     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 612         long l4SrcPortField;
 613         long l4DstPortField;
 614         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 615
 616         if (protocol == NwConstants.IP_PROT_TCP) {
 617             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 618             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 619         } else {
 620             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 621             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 622         }
 623         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 624                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 625                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 626                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 627                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 628                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 629                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 630                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 631                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 632                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 633                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 634                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 635                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 636                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 637                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 638
 639         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 640                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 641     }
 642
 643     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 644         List<MatchInfo> matches = new ArrayList<>();
 645         matches.add(MatchEthernetType.IPV4);
 646         matches.add(MatchIpProtocol.ICMP);
 647         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 648
 649         List<ActionInfo> actionInfos = new ArrayList<>();
 650         actionInfos.add(new ActionDrop());
 651
 652         List<InstructionInfo> instructions = new ArrayList<>();
 653         instructions.add(new InstructionApplyActions(actionInfos));
 654
 655         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 656                 NwConstants.IP_PROT_ICMP);
 657         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 658                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 659                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 660         return flow;
 661     }
 662
 663     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 664         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 665         BigInteger cookie = getCookieOutboundFlow(routerId);
 666         List<MatchInfo> matches = new ArrayList<>();
 667         matches.add(MatchEthernetType.IPV4);
 668         matches.add(new MatchIpProtocol((short)protocol));
 669         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 670
 671         List<InstructionInfo> instructions = new ArrayList<>();
 672         List<ActionInfo> actionsInfos = new ArrayList<>();
 673         actionsInfos.add(new ActionPuntToController());
 674         if (snatPuntTimeout != 0) {
 675             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 676         }
 677         instructions.add(new InstructionApplyActions(actionsInfos));
 678
 679         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 680         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 681             5, flowRef, 0, 0,
 682             cookie, matches, instructions);
 683         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 684         return flowEntity;
 685     }
 686
 687     public void createOutboundTblEntry(BigInteger dpnId, long routerId, WriteTransaction writeFlowInvTx) {
 688         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 689         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 690         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 691         mdsalManager.addFlowToTx(tcpflowEntity, writeFlowInvTx);
 692
 693         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 694         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 695         mdsalManager.addFlowToTx(udpflowEntity, writeFlowInvTx);
 696
 697         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 698         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 699         mdsalManager.addFlowToTx(icmpDropFlow, writeFlowInvTx);
 700     }
 701
 702     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 703         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 704         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 705         try {
 706             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 707                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 708                     .setSourceDpid(srcDpId)
 709                     .setDestinationDpid(dstDpId)
 710                     .setTunnelType(tunType)
 711                     .build());
 712             rpcResult = result.get();
 713             if (!rpcResult.isSuccessful()) {
 714                 tunType = TunnelTypeGre.class;
 715                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 716                     .setSourceDpid(srcDpId)
 717                     .setDestinationDpid(dstDpId)
 718                     .setTunnelType(tunType)
 719                     .build());
 720                 rpcResult = result.get();
 721                 if (!rpcResult.isSuccessful()) {
 722                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 723                             rpcResult.getErrors());
 724                 } else {
 725                     return rpcResult.getResult().getInterfaceName();
 726                 }
 727                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 728                         rpcResult.getErrors());
 729             } else {
 730                 return rpcResult.getResult().getInterfaceName();
 731             }
 732         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 733             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 734                     + "between {} and {}", srcDpId, dstDpId, e);
 735         }
 736
 737         return null;
 738     }
 739
 740     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 741                                                     WriteTransaction writeFlowInvTx) {
 742         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 743         // Install miss entry pointing to group
 744         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 745         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 746     }
 747
 748     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 749             String routerName, long routerId) {
 750         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 751             dpnId, bucketInfo.get(0));
 752         // Install the select group
 753         long groupId = createGroupId(getGroupIdKey(routerName));
 754         GroupEntity groupEntity =
 755             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 756         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 757         mdsalManager.syncInstallGroup(groupEntity);
 758         // Install miss entry pointing to group
 759         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 760         if (flowEntity == null) {
 761             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 762                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 763                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 764             return;
 765         }
 766         mdsalManager.installFlow(flowEntity);
 767     }
 768
 769     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 770         long groupId = createGroupId(getGroupIdKey(routerName));
 771         GroupEntity groupEntity =
 772             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 773         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 774         mdsalManager.syncInstallGroup(groupEntity);
 775         return groupId;
 776     }
 777
 778     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 779         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 780                 dpId, routerName, groupId);
 781         List<MatchInfo> matches = new ArrayList<>();
 782         matches.add(MatchEthernetType.IPV4);
 783         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 784
 785         List<ActionInfo> actionsInfo = new ArrayList<>();
 786         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 787                 routerName);
 788         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 789         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 790         actionsInfo.add(new ActionGroup(groupId));
 791         List<InstructionInfo> instructions = new ArrayList<>();
 792         instructions.add(new InstructionApplyActions(actionsInfo));
 793         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 794         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 795                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 796                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 797
 798         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 799         return flowEntity;
 800     }
 801
 802     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 803
 804         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 805             routerName);
 806         List<MatchInfo> matches = new ArrayList<>();
 807         matches.add(MatchEthernetType.IPV4);
 808         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 809
 810         List<InstructionInfo> instructions = new ArrayList<>();
 811         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 812
 813         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 814         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 815             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 816             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 817
 818         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 819         return flowEntity;
 820     }
 821
 822     // TODO : Replace this with ITM Rpc once its available with full functionality
 823     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 824                                                      WriteTransaction writeFlowInvTx) {
 825         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 826             dpnId, routerName);
 827         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 828         if (flowEntity == null) {
 829             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 830                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 831                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 832                     dpnId, routerName);
 833             return;
 834         }
 835         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 836
 837     }
 838
 839     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 840         List<MatchInfo> matches = new ArrayList<>();
 841         matches.add(MatchEthernetType.IPV4);
 842         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
 843                 routerName);
 844         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 845         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 846         List<InstructionInfo> instructions = new ArrayList<>();
 847         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 848                 MetaDataUtil.METADATA_MASK_VRFID));
 849         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 850         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 851                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 852                 instructions);
 853         return flowEntity;
 854     }
 855
 856     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 857         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 858                 .FLOWID_SEPARATOR + routerID;
 859     }
 860
 861     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 862         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 863                 .FLOWID_SEPARATOR + routerID;
 864     }
 865
 866     private String getGroupIdKey(String routerName) {
 867         return "snatmiss." + routerName;
 868     }
 869
 870     protected long createGroupId(String groupIdKey) {
 871         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 872             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 873             .build();
 874         try {
 875             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 876             RpcResult<AllocateIdOutput> rpcResult = result.get();
 877             return rpcResult.getResult().getIdValue();
 878         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 879             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 880         }
 881         return 0;
 882     }
 883
 884     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 885         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 886         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 887         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 888         List<BucketInfo> listBucketInfo = new ArrayList<>();
 889
 890         if (ifNamePrimary != null) {
 891             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 892             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 893                     interfaceManager, ifNamePrimary, routerId);
 894             if (listActionInfoPrimary.isEmpty()) {
 895                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 896                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 897                         ifNamePrimary);
 898             }
 899         } else {
 900             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 901                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 902         }
 903         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 904
 905         listBucketInfo.add(0, bucketPrimary);
 906         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 907     }
 908
 909     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 910                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 911         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 912         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 913         List<BucketInfo> listBucketInfo = new ArrayList<>();
 914
 915         if (ifNamePrimary != null) {
 916             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 917                     ifNamePrimary);
 918             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 919                     interfaceManager, ifNamePrimary, routerId);
 920             if (listActionInfoPrimary.isEmpty()) {
 921                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 922                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 923                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 924             }
 925         } else {
 926             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 927                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 928         }
 929         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 930
 931         listBucketInfo.add(0, bucketPrimary);
 932         return listBucketInfo;
 933     }
 934
 935     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 936             WriteTransaction writeFlowInvTx) {
 937        /*
 938         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 939         */
 940
 941         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 942
 943 /*
 944         List<BucketInfo> listBucketInfo = new ArrayList<>();
 945         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 946         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 947         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 948         listBucketInfo.add(0, bucketPrimary);
 949 */
 950
 951         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, writeFlowInvTx);
 952         installTerminatingServiceTblEntry(dpnId, routerName, routerId, writeFlowInvTx);
 953         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 954         installNaptPfibEntry(dpnId, routerId, writeFlowInvTx);
 955         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 956         if (networkId != null) {
 957             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 958             if (vpnUuid != null) {
 959                 Long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 960                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 961                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 962                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 963                     if (vpnId != null && vpnId != NatConstants.INVALID_ID) {
 964                         installNaptPfibEntry(dpnId, vpnId, null);
 965                     }
 966                     return Collections.emptyList();
 967                 });
 968             } else {
 969                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 970             }
 971         } else {
 972             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 973         }
 974     }
 975
 976     List<BucketInfo> getBucketInfoForPrimaryNaptSwitch() {
 977         List<BucketInfo> listBucketInfo = new ArrayList<>();
 978         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 979         listActionInfoPrimary.add(new ActionNxResubmit(NwConstants.INTERNAL_TUNNEL_TABLE));
 980         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 981         listBucketInfo.add(0, bucketPrimary);
 982         return listBucketInfo;
 983     }
 984
 985     public void installNaptPfibEntry(BigInteger dpnId, long segmentId, WriteTransaction writeFlowInvTx) {
 986         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 987         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 988         if (writeFlowInvTx != null) {
 989             mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
 990         } else {
 991             mdsalManager.installFlow(naptPfibFlowEntity);
 992         }
 993     }
 994
 995     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 996
 997         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 998         List<MatchInfo> matches = new ArrayList<>();
 999         matches.add(MatchEthernetType.IPV4);
1000         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
1001
1002         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
1003         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
1004         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1005         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1006         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1007
1008         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1009         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1010             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1011             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1012         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1013         return flowEntity;
1014     }
1015
1016     public void handleSnatReverseTraffic(BigInteger dpnId, Routers router, long routerId, String routerName,
1017             String externalIp, WriteTransaction writeFlowInvTx) {
1018         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1019             dpnId, routerId, externalIp);
1020         Uuid networkId = router.getNetworkId();
1021         if (networkId == null) {
1022             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1023             return;
1024         }
1025         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1026         if (vpnName == null) {
1027             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1028                     + "configuration {} in router {}", networkId, externalIp, routerId);
1029             return;
1030         }
1031         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1032             externalIp, networkId, router, writeFlowInvTx);
1033         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1034             dpnId, routerId, externalIp);
1035     }
1036
1037     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1038                                                 final long routerId, final String routerName, final String externalIp,
1039                                                 final Uuid extNetworkId, final Routers router,
1040                                                 final WriteTransaction writeFlowInvTx) {
1041         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1042                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1043         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1044         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1045         if (rd == null || rd.isEmpty()) {
1046             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1047             return;
1048         }
1049         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1050         if (extNwProvType == null) {
1051             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1052             return;
1053         }
1054         if (extNwProvType == ProviderTypes.VXLAN) {
1055             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1056                 tx -> evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp,
1057                         vpnName, rd, nextHopIp, tx, routerId, routerName, writeFlowInvTx)), LOG,
1058                 "Error installing FIB and TS flows");
1059             return;
1060         }
1061         //Generate VPN label for the external IP
1062         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1063             .setIpPrefix(externalIp).build();
1064         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1065
1066         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1067         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1068             if (result.isSuccessful()) {
1069                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1070                 GenerateVpnLabelOutput output = result.getResult();
1071                 final long label = output.getLabel();
1072
1073                 int externalIpInDsFlag = 0;
1074                 //Get IPMaps from the DB for the router ID
1075                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1076                 if (dbIpMaps != null) {
1077                     for (IpMap dbIpMap : dbIpMaps) {
1078                         String dbExternalIp = dbIpMap.getExternalIp();
1079                         //Select the IPMap, whose external IP is the IP for which FIB is installed
1080                         if (dbExternalIp.contains(externalIp)) {
1081                             String dbInternalIp = dbIpMap.getInternalIp();
1082                             IpMapKey dbIpMapKey = dbIpMap.key();
1083                             LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1084                                     + "and externalIp {}", label, dbInternalIp, externalIp);
1085                             IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1086                                 .setExternalIp(dbExternalIp).setLabel(label).build();
1087                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1088                                 naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1089                             externalIpInDsFlag++;
1090                         }
1091                     }
1092                     if (externalIpInDsFlag <= 0) {
1093                         LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1094                                 + "Failed to update label {} for routerId {} in DS",
1095                                 externalIp, label, routerId);
1096                         String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1097                             + " found in DS for router %s", label, externalIp, routerId);
1098                         return Futures.immediateFailedFuture(new Exception(errMsg));
1099                     }
1100                 } else {
1101                     LOG.error("advToBgpAndInstallFibAndTsFlows : Failed to write label {} for externalIp {} for"
1102                             + " routerId {} in DS", label, externalIp, routerId);
1103                 }
1104                 //Inform BGP
1105                 long l3vni = 0;
1106                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1107                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
1108                 }
1109                 Routers extRouter = router != null ? router :
1110                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1111                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
1112                         extRouter);
1113                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd, externalSubnetId,
1114                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1115                     RouteOrigin.STATIC, dpnId);
1116
1117                 //Install custom FIB routes
1118                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1119                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1120                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, writeFlowInvTx,
1121                         extNwProvType);
1122                 makeLFibTableEntry(dpnId, label, tableId, writeFlowInvTx);
1123
1124                 //Install custom FIB routes - FIB table.
1125                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1126                         routerName, externalIp);
1127                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1128                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1129                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId,
1130                             NwConstants.INBOUND_NAPT_TABLE,writeFlowInvTx);
1131                 }
1132                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1133                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1134                         externalSubnetId);
1135                 String externalVpn = vpnName;
1136                 if (externalSubnet.isPresent()) {
1137                     externalVpn =  externalSubnetId.getValue();
1138                 }
1139                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1140                     .setVpnName(externalVpn)
1141                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1142                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1143                     .setInstruction(fibTableCustomInstructions).build();
1144                 return fibService.createFibEntry(input);
1145             } else {
1146                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1147                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1148                     externalIp, vpnName, result.getErrors());
1149                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1150             }
1151         }, MoreExecutors.directExecutor());
1152
1153         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1154
1155             @Override
1156             public void onFailure(@Nonnull Throwable error) {
1157                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1158             }
1159
1160             @Override
1161             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
1162                 if (result.isSuccessful()) {
1163                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1164                             externalIp);
1165                 } else {
1166                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1167                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1168                 }
1169             }
1170         }, MoreExecutors.directExecutor());
1171     }
1172
1173     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1174             String externalIp) {
1175         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1176         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1177         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1178                 externalIp, router);
1179         int instructionIndex = 0;
1180         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1181             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1182             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1183                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1184             instructionIndex++;
1185         }
1186
1187         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1188         return fibTableCustomInstructions;
1189     }
1190
1191     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId, WriteTransaction writeFlowInvTx) {
1192         List<MatchInfo> matches = new ArrayList<>();
1193         matches.add(MatchEthernetType.MPLS_UNICAST);
1194         matches.add(new MatchMplsLabel(serviceId));
1195
1196         List<Instruction> instructions = new ArrayList<>();
1197         List<ActionInfo> actionsInfos = new ArrayList<>();
1198         actionsInfos.add(new ActionPopMpls());
1199         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1200         instructions.add(writeInstruction);
1201         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1202
1203         // Install the flow entry in L3_LFIB_TABLE
1204         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1205
1206         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1207             10, flowRef, 0, 0,
1208             COOKIE_VM_LFIB_TABLE, matches, instructions);
1209
1210         mdsalManager.addFlowToTx(dpId, flowEntity, writeFlowInvTx);
1211
1212         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1213     }
1214
1215     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1216              List<Instruction> customInstructions, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
1217         List<MatchInfo> mkMatches = new ArrayList<>();
1218
1219         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1220             dpnId, serviceId, customInstructions);
1221
1222         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1223             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1224         } else {
1225             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1226         }
1227
1228         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1229             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1230             String.format("%s:%d", "TST Flow Entry ", serviceId),
1231             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1232
1233         mdsalManager.addFlowToTx(dpnId, terminatingServiceTableFlowEntity, writeFlowInvTx);
1234     }
1235
1236     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1237         InstanceIdentifier<RouterIds> id = InstanceIdentifier.builder(
1238             RouterIdName.class).child(RouterIds.class, new RouterIdsKey(routerId)).build();
1239         return id;
1240     }
1241
1242     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1243         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1244                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1245     }
1246
1247     @Override
1248     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1249         String routerName = original.getRouterName();
1250         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1251         if (routerId == NatConstants.INVALID_ID) {
1252             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1253             return;
1254         }
1255         // Check if its update on SNAT flag
1256         boolean originalSNATEnabled = original.isEnableSnat();
1257         boolean updatedSNATEnabled = update.isEnableSnat();
1258         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1259             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1260         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1261          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1262          */
1263         long bgpVpnId = NatConstants.INVALID_ID;
1264         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1265         if (bgpVpnUuid != null) {
1266             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1267         }
1268         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1269         if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1270             // Router has no interface attached
1271             return;
1272         }
1273         final long finalBgpVpnId = bgpVpnId;
1274         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1275             List<ListenableFuture<Void>> futures = new ArrayList<>();
1276             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeFlowInvTx -> {
1277                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(removeFlowInvTx -> {
1278                     Uuid networkId = original.getNetworkId();
1279                     if (originalSNATEnabled != updatedSNATEnabled) {
1280                         if (originalSNATEnabled) {
1281                             //SNAT disabled for the router
1282                             Uuid networkUuid = original.getNetworkId();
1283                             LOG.info("update : SNAT disabled for Router {}", routerName);
1284                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1285                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1286                                     removeFlowInvTx);
1287                         } else {
1288                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1289                             handleEnableSnat(original, routerId, dpnId, finalBgpVpnId, removeFlowInvTx);
1290                         }
1291                     }
1292                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1293                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1294                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1295                     }
1296
1297                     //Check if the Update is on External IPs
1298                     LOG.debug("update : Checking if this is update on External IPs");
1299                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1300                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1301
1302                     //Check if the External IPs are added during the update.
1303                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1304                     addedExternalIps.removeAll(originalExternalIps);
1305                     if (addedExternalIps.size() != 0) {
1306                         LOG.debug("update : Start processing of the External IPs addition during the update "
1307                                 + "operation");
1308                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1309                                 update.getExtGwMacAddress(), dpnId,
1310                                 update.getNetworkId(), null);
1311
1312                         for (String addedExternalIp : addedExternalIps) {
1313                 /*
1314                     1) Do nothing in the IntExtIp model.
1315                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1316                  */
1317                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1318                             String externalIp = externalIpParts[0];
1319                             String externalIpPrefix = externalIpParts[1];
1320                             String externalpStr = externalIp + "/" + externalIpPrefix;
1321                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1322                                             + "router ID {} in the ExternalIpsCounter model.",
1323                                     externalpStr, routerId);
1324                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1325                         }
1326                         LOG.debug(
1327                                 "update : End processing of the External IPs addition during the update operation");
1328                     }
1329
1330                     //Check if the External IPs are removed during the update.
1331                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1332                     removedExternalIps.removeAll(updatedExternalIps);
1333                     if (removedExternalIps.size() > 0) {
1334                         LOG.debug("update : Start processing of the External IPs removal during the update "
1335                                 + "operation");
1336                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1337                                 removedExternalIps, original.getExtGwMacAddress(),
1338                                 dpnId, networkId);
1339
1340                         for (String removedExternalIp : removedExternalIps) {
1341                 /*
1342                     1) Remove the mappings in the IntExt IP model which has external IP.
1343                     2) Remove the external IP in the ExternalCounter model.
1344                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1345                        least loaded external IP.
1346                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1347                     4) Increase the count of the allocated external IP by one.
1348                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1349                      i.e. the route for the external IP is absent.
1350                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1351                      the removed external IPs and also from the model.
1352                     7) Advertise to the BGP for removing the route for the removed external IPs.
1353                  */
1354
1355                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1356                             String externalIp = externalIpParts[0];
1357                             String externalIpPrefix = externalIpParts[1];
1358                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1359
1360                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1361                                     + "entries for removed external IP {}", externalIpAddrStr);
1362                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1363                             String vpnName = "";
1364                             if (vpnUuId != null) {
1365                                 vpnName = vpnUuId.getValue();
1366                             }
1367                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1368                                     update.getExtGwMacAddress(), removeFlowInvTx);
1369
1370                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1371                             //Get the internal IPs which are associated to the removed external IPs
1372                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1373                             List<String> removedInternalIps = new ArrayList<>();
1374                             for (IpMap ipMap : ipMaps) {
1375                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1376                                     removedInternalIps.add(ipMap.getInternalIp());
1377                                 }
1378                             }
1379
1380                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1381                             for (String removedInternalIp : removedInternalIps) {
1382                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1383                                                 + "router ID {} from the IntExtIP model",
1384                                         removedInternalIp, routerId);
1385                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1386                             }
1387
1388                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1389                                             + "router ID {} from the ExternalIpsCounter model.",
1390                                     externalIpAddrStr, routerId);
1391                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1392
1393                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1394                                     + "whose external IPs were removed.");
1395                             for (String removedInternalIp : removedInternalIps) {
1396                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1397                                         removedInternalIp, writeFlowInvTx);
1398                             }
1399
1400                             LOG.debug("update : Remove the NAPT translation entries from "
1401                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1402                             //Get the internalIP and internal Port which were associated to the removed external IP.
1403                             List<Integer> externalPorts = new ArrayList<>();
1404                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1405                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1406                                     .builder(IntextIpPortMap.class)
1407                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1408                             Optional<IpPortMapping> ipPortMapping =
1409                                     MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1410                             if (ipPortMapping.isPresent()) {
1411                                 List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.get()
1412                                         .getIntextIpProtocolType();
1413                                 for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
1414                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1415                                     List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
1416                                     for (IpPortMap ipPortMap : ipPortMaps) {
1417                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1418                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1419                                             externalPorts.add(ipPortExternal.getPortNum());
1420                                             List<String> removedInternalIpPorts =
1421                                                     protoTypesIntIpPortsMap.get(protoType);
1422                                             if (removedInternalIpPorts != null) {
1423                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1424                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1425                                             } else {
1426                                                 removedInternalIpPorts = new ArrayList<>();
1427                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1428                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1429                                             }
1430                                         }
1431                                     }
1432                                 }
1433                             }
1434
1435                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1436                             // the removed external IP.
1437                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1438                                     protoTypesIntIpPortsMap.entrySet();
1439                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1440                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1441                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1442                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1443                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1444                                     // Remove the IP port map from the intext-ip-port-map model,
1445                                     // which were containing the removed external IP
1446                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1447                                             protocolType);
1448                                     //Remove the IP port incomint packer map.
1449                                     naptPacketInHandler.removeIncomingPacketMap(
1450                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1451                                     String[] removedInternalIpPortParts = removedInternalIpPort
1452                                             .split(NatConstants.COLON_SEPARATOR);
1453                                     if (removedInternalIpPortParts.length == 2) {
1454                                         String removedInternalIp = removedInternalIpPortParts[0];
1455                                         String removedInternalPort = removedInternalIpPortParts[1];
1456                                         List<String> removedInternalPortsList =
1457                                                 internalIpPortMap.get(removedInternalPort);
1458                                         if (removedInternalPortsList != null) {
1459                                             removedInternalPortsList.add(removedInternalPort);
1460                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1461                                         } else {
1462                                             removedInternalPortsList = new ArrayList<>();
1463                                             removedInternalPortsList.add(removedInternalPort);
1464                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1465                                         }
1466                                     }
1467                                 }
1468                             }
1469
1470                             // Delete the entry from SnatIntIpPortMap DS
1471                             Set<String> internalIps = internalIpPortMap.keySet();
1472                             for (String internalIp : internalIps) {
1473                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1474                                         + "model SnatIntIpPortMap", internalIp);
1475                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1476                             }
1477
1478                             naptManager.removeNaptPortPool(externalIp);
1479
1480                             LOG.debug("update : Remove the NAPT translation entries from Inbound NAPT tables for "
1481                                     + "the removed external IP {}", externalIp);
1482                             for (Integer externalPort : externalPorts) {
1483                                 //Remove the NAPT translation entries from Inbound NAPT table
1484                                 naptEventHandler.removeNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE,
1485                                         routerId, externalIp, externalPort);
1486                             }
1487
1488                             Set<Map.Entry<String, List<String>>> internalIpPorts = internalIpPortMap.entrySet();
1489                             for (Map.Entry<String, List<String>> internalIpPort : internalIpPorts) {
1490                                 String internalIp = internalIpPort.getKey();
1491                                 LOG.debug("update : Remove the NAPT translation entries from Outbound NAPT tables "
1492                                         + "for the removed internal IP {}", internalIp);
1493                                 List<String> internalPorts = internalIpPort.getValue();
1494                                 for (String internalPort : internalPorts) {
1495                                     //Remove the NAPT translation entries from Outbound NAPT table
1496                                     naptPacketInHandler.removeIncomingPacketMap(
1497                                             routerId + NatConstants.COLON_SEPARATOR + internalIp
1498                                                     + NatConstants.COLON_SEPARATOR + internalPort);
1499                                     naptEventHandler.removeNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1500                                             routerId, internalIp, Integer.parseInt(internalPort));
1501                                 }
1502                             }
1503                         }
1504                         LOG.debug(
1505                                 "update : End processing of the External IPs removal during the update operation");
1506                     }
1507
1508                     //Check if its Update on subnets
1509                     LOG.debug("update : Checking if this is update on subnets");
1510                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1511                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1512                     Set<Uuid> addedSubnetIds = new HashSet<>(updatedSubnetIds);
1513                     addedSubnetIds.removeAll(originalSubnetIds);
1514
1515                     //Check if the Subnet IDs are added during the update.
1516                     if (addedSubnetIds.size() != 0) {
1517                         LOG.debug(
1518                                 "update : Start processing of the Subnet IDs addition during the update operation");
1519                         for (Uuid addedSubnetId : addedSubnetIds) {
1520                 /*
1521                     1) Select the least loaded external IP for the subnet and store the mapping of the
1522                     subnet IP and the external IP in the IntExtIp model.
1523                     2) Increase the count of the selected external IP by one.
1524                     3) Advertise to the BGP if external IP is allocated for the first time for the
1525                     router i.e. the route for the external IP is absent.
1526                  */
1527                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1528                             if (subnetIp != null) {
1529                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1530                                         writeFlowInvTx);
1531                             }
1532                         }
1533                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1534                     }
1535
1536                     //Check if the Subnet IDs are removed during the update.
1537                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1538                     removedSubnetIds.removeAll(updatedSubnetIds);
1539                     if (removedSubnetIds.size() != 0) {
1540                         LOG.debug(
1541                                 "update : Start processing of the Subnet IDs removal during the update operation");
1542                         for (Uuid removedSubnetId : removedSubnetIds) {
1543                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1544                             if (subnetAddr != null) {
1545                     /*
1546                         1) Remove the subnet IP and the external IP in the IntExtIp map
1547                         2) Decrease the count of the coresponding external IP by one.
1548                         3) Advertise to the BGP for removing the routes of the corresponding external
1549                         IP if its not allocated to any other internal IP.
1550                     */
1551
1552                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1553                                         subnetAddr[0] + "/" + subnetAddr[1]);
1554                                 if (externalIp == null) {
1555                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1556                                             routerId, subnetAddr[0]);
1557                                     return;
1558                                 }
1559
1560                                 naptManager.updateCounter(routerId, externalIp, false);
1561                                 // Traverse entire model of external-ip counter whether external ip is not
1562                                 // used by any other internal ip in any router
1563                                 if (!isExternalIpAllocated(externalIp)) {
1564                                     LOG.debug("update : external ip is not allocated to any other "
1565                                             + "internal IP so proceeding to remove routes");
1566                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1567                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1568                                             removeFlowInvTx);
1569                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1570                                                     + "router {} with networkId {} and externalIp {}",
1571                                             dpnId, routerId, networkId, externalIp);
1572                                 }
1573
1574                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1575                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1576                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1577                             }
1578                         }
1579                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1580                     }
1581                 }));
1582             }));
1583             return futures;
1584         }, NatConstants.NAT_DJC_MAX_RETRIES);
1585     }
1586
1587     private boolean isExternalIpAllocated(String externalIp) {
1588         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1589         Optional<ExternalIpsCounter> externalCountersData =
1590             MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
1591         if (externalCountersData.isPresent()) {
1592             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1593             List<ExternalCounters> externalCounters = externalIpsCounters.getExternalCounters();
1594             for (ExternalCounters ext : externalCounters) {
1595                 for (ExternalIpCounter externalIpCount : ext.getExternalIpCounter()) {
1596                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1597                         if (externalIpCount.getCounter() != 0) {
1598                             return true;
1599                         }
1600                         break;
1601                     }
1602                 }
1603             }
1604         }
1605         return false;
1606     }
1607
1608     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1609             Uuid networkId, String subnetIp, WriteTransaction writeFlowInvTx) {
1610         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1611         try {
1612             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1613             if (address instanceof Inet6Address) {
1614                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1615                 return;
1616             }
1617         } catch (UnknownHostException e) {
1618             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1619             return;
1620         }
1621         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1622         if (leastLoadedExtIpAddr != null) {
1623             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1624             String leastLoadedExtIp = externalIpParts[0];
1625             String leastLoadedExtIpPrefix = externalIpParts[1];
1626             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1627             subnetIp = subnetIpParts[0];
1628             String subnetIpPrefix = subnetIpParts[1];
1629             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1630             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1631                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1632                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1633             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1634
1635
1636             // Check if external IP is already assigned a route. (i.e. External IP is previously
1637             // allocated to any of the subnets)
1638             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1639             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1640             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1641             if (label != null) {
1642                 //update
1643                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1644                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1645                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1646                     label, internalIp, leastLoadedExtIpAddrStr);
1647                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1648                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1649                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1650                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1651                 return;
1652             }
1653
1654             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1655             // for the first time and hence not having a route.
1656             //Get the VPN Name using the network ID
1657             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1658             if (vpnName != null) {
1659                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1660                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1661                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1662                         + "added after gateway-set");
1663                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1664                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1665                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1666                         return;
1667                     }
1668                 }
1669                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1670                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1671                     writeFlowInvTx);
1672             }
1673         }
1674     }
1675
1676     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1677         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1678         for (IpMap ipMap : ipMaps) {
1679             if (ipMap.getExternalIp().equals(externalIp)) {
1680                 if (ipMap.getLabel() != null) {
1681                     return ipMap.getLabel();
1682                 }
1683             }
1684         }
1685         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1686         return null;
1687     }
1688
1689     @Override
1690     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1691         LOG.trace("remove : Router delete method");
1692         /*
1693         ROUTER DELETE SCENARIO
1694         1) Get the router ID from the event.
1695         2) Build the cookie information from the router ID.
1696         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1697         4) Build the flow with the cookie value.
1698         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1699         6) Remove the flows from the other switches which points to the primary and secondary
1700          switches for the flows related the router ID.
1701         7) Get the list of external IP address maintained for the router ID.
1702         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1703         9) Withdraw the corresponding routes from the BGP.
1704          */
1705
1706         if (identifier == null || router == null) {
1707             LOG.error("remove : returning without processing since routers is null");
1708             return;
1709         }
1710
1711         String routerName = router.getRouterName();
1712         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1713             () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
1714                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1715                         routerName);
1716                 List<ListenableFuture<Void>> futures = new ArrayList<>();
1717                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1718                 if (routerId == NatConstants.INVALID_ID) {
1719                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1720                             routerName);
1721                     return;
1722                 }
1723                 long bgpVpnId = NatConstants.INVALID_ID;
1724                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1725                 if (bgpVpnUuid != null) {
1726                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1727                 }
1728                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1729                         tx);
1730                 Uuid networkUuid = router.getNetworkId();
1731
1732                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1733                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1734                     // No NAPT switch for external router, probably because the router is not attached to
1735                     // any
1736                     // internal networks
1737                     LOG.debug(
1738                             "No NAPT switch for router {}, check if router is attached to any internal "
1739                                     + "network",
1740                             routerName);
1741                     return;
1742                 } else {
1743                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1744                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1745                             routerId, tx);
1746                 }
1747                 NatOverVxlanUtil.releaseVNI(routerName, idManager);
1748             })), NatConstants.NAT_DJC_MAX_RETRIES);
1749     }
1750
1751     // TODO Clean up the exception handling
1752     @SuppressWarnings("checkstyle:IllegalCatch")
1753     public void handleDisableSnat(Routers router, Uuid networkUuid, @Nonnull Collection<String> externalIps,
1754                                   boolean routerFlag, String vpnName, BigInteger naptSwitchDpnId,
1755                                   long routerId, WriteTransaction removeFlowInvTx) {
1756         LOG.info("handleDisableSnat : Entry");
1757         String routerName = router.getRouterName();
1758         try {
1759             if (routerFlag) {
1760                 removeNaptSwitch(routerName);
1761             } else {
1762                 updateNaptSwitch(routerName, BigInteger.ZERO);
1763             }
1764
1765             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1766             naptManager.removeExternalCounter(routerId);
1767
1768             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1769             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1770                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1771                     + "router ID {} from RouterNaptSwitch model", routerId);
1772                 return;
1773             }
1774             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1775                     networkUuid);
1776             if (extNwProvType == null) {
1777                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1778                 return;
1779             }
1780             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1781             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1782                     externalSubnetList, removeFlowInvTx, extNwProvType);
1783             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1784             try {
1785                 String externalSubnetVpn = null;
1786                 for (Uuid externalSubnetId : externalSubnetList) {
1787                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1788                     // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1789                     if (externalSubnet.isPresent()) {
1790                         externalSubnetVpn =  externalSubnetId.getValue();
1791                         clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1792                                 router.getExtGwMacAddress(), removeFlowInvTx);
1793                     }
1794                 }
1795                 if (externalSubnetVpn == null) {
1796                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1797                             router.getExtGwMacAddress(), removeFlowInvTx);
1798                 }
1799             } catch (Exception ex) {
1800                 LOG.error("handleDisableSnat : Failed to remove fib entries for routerId {} in naptSwitchDpnId {}",
1801                     routerId, naptSwitchDpnId, ex);
1802             }
1803             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1804             // for the router ID.
1805             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1806                     + "router ID {} in the DS", routerId);
1807             naptManager.removeMapping(routerId);
1808         } catch (Exception ex) {
1809             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, ex);
1810         }
1811         LOG.info("handleDisableSnat : Exit");
1812     }
1813
1814     // TODO Clean up the exception handling
1815     @SuppressWarnings("checkstyle:IllegalCatch")
1816     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1817                                              @Nonnull Collection<String> externalIps,
1818                                              String vpnId, WriteTransaction writeFlowInvTx) {
1819         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1820                 + "with internet vpn {}", routerName, vpnId);
1821         try {
1822             BigInteger naptSwitchDpnId = null;
1823             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1824                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1825             Optional<RouterToNaptSwitch> rtrToNapt =
1826                 read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1827             if (rtrToNapt.isPresent()) {
1828                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1829             }
1830             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1831
1832             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1833                     writeFlowInvTx);
1834             try {
1835                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1836                 if (extGwMacAddress != null) {
1837                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1838                             + "External Router ID {}", extGwMacAddress, routerId);
1839                 } else {
1840                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1841                             + "External Router ID {}", routerId);
1842                     return;
1843                 }
1844                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1845                         writeFlowInvTx);
1846             } catch (Exception ex) {
1847                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1848                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1849             }
1850             NatOverVxlanUtil.releaseVNI(vpnId, idManager);
1851         } catch (Exception ex) {
1852             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1853                     + "with internet vpn {}", routerName, vpnId, ex);
1854         }
1855         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1856                 routerName, vpnId);
1857     }
1858
1859     // TODO Clean up the exception handling
1860     @SuppressWarnings("checkstyle:IllegalCatch")
1861     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1862         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1863             .setPrimarySwitchId(naptSwitchId).build();
1864         try {
1865             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1866                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1867         } catch (Exception ex) {
1868             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1869                 naptSwitchId, routerName);
1870         }
1871         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1872             naptSwitchId, routerName);
1873     }
1874
1875     protected void removeNaptSwitch(String routerName) {
1876         // Remove router and switch from model
1877         InstanceIdentifier<RouterToNaptSwitch> id =
1878             InstanceIdentifier.builder(NaptSwitches.class)
1879                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1880         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1881         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1882         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1883         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1884     }
1885
1886     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1887                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1888                                                 @Nonnull Collection<String> externalIps,
1889                                                 Collection<Uuid> externalSubnetList,
1890                                                 WriteTransaction removeFlowInvTx, ProviderTypes extNwProvType) {
1891         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1892         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1893
1894         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1895         // traffic which comes from the  VMs of the NAPT switches)
1896         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1897         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1898
1899         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1900                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1901         mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
1902
1903         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1904         // traffic which comes from the VMs of the non NAPT switches)
1905         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, routerId,
1906                 routerName);
1907         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1908         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1909         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1910                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1911         mdsalManager.removeFlowToTx(tsNatFlowEntity, removeFlowInvTx);
1912
1913         //Remove the flow table 25->44 from NAPT Switch
1914         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1915             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
1916         }
1917
1918         //Remove the Outbound flow entry which forwards the packet to FIB Table
1919         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1920                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1921
1922         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1923                 NwConstants.IP_PROT_TCP);
1924         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1925             outboundTcpNatFlowRef);
1926         mdsalManager.removeFlowToTx(outboundTcpNatFlowEntity, removeFlowInvTx);
1927
1928         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1929                 NwConstants.IP_PROT_UDP);
1930         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1931                 outboundUdpNatFlowRef);
1932         mdsalManager.removeFlowToTx(outboundUdpNatFlowEntity, removeFlowInvTx);
1933
1934         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1935                 NwConstants.IP_PROT_ICMP);
1936         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1937                 icmpDropFlowRef);
1938         mdsalManager.removeFlowToTx(icmpDropFlowEntity, removeFlowInvTx);
1939
1940         removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, removeFlowInvTx);
1941         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1942         // External Subnet Vpn Id.
1943         for (Uuid externalSubnetId : externalSubnetList) {
1944             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1945             if (subnetVpnId != -1) {
1946                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1947                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1948                         natPfibSubnetFlowRef);
1949                 mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1950                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1951                         + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1952                         subnetVpnId, dpnId);
1953             }
1954         }
1955
1956         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1957         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1958         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1959
1960         LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1961                 + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1962         mdsalManager.removeFlowToTx(natPfibFlowEntity, removeFlowInvTx);
1963
1964         // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1965         // - This does not work since ext-routers is deleted already - no network info
1966         //Get the VPN ID from the ExternalNetworks model
1967         long vpnId = -1;
1968         if (vpnName == null || vpnName.isEmpty()) {
1969             // ie called from router delete cases
1970             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1971             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1972             if (vpnUuid != null) {
1973                 vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
1974                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete or "
1975                         + "disableSNAT scenario", vpnId, networkId);
1976             }
1977         } else {
1978             // ie called from disassociate vpn case
1979             LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
1980                     vpnName);
1981             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
1982             LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}", vpnId);
1983         }
1984
1985         if (vpnId != NatConstants.INVALID_ID) {
1986             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
1987             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
1988             FlowEntity natPfibVpnFlowEntity =
1989                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
1990             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the DPN ID {} "
1991                     + "and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
1992             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, removeFlowInvTx);
1993         }
1994
1995         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
1996         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
1997         if (ipPortMapping == null) {
1998             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
1999             return;
2000         }
2001
2002         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2003         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2004             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2005             for (IpPortMap ipPortMap : ipPortMaps) {
2006                 String ipPortInternal = ipPortMap.getIpPortInternal();
2007                 String[] ipPortParts = ipPortInternal.split(":");
2008                 if (ipPortParts.length != 2) {
2009                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2010                     return;
2011                 }
2012                 String internalIp = ipPortParts[0];
2013                 String internalPort = ipPortParts[1];
2014
2015                 //Build the flow for the outbound NAPT table
2016                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2017                         + NatConstants.COLON_SEPARATOR + internalPort);
2018                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2019                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2020                 FlowEntity outboundNaptFlowEntity =
2021                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2022
2023                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2024                         + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2025                 mdsalManager.removeFlowToTx(outboundNaptFlowEntity, removeFlowInvTx);
2026
2027                 IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2028                 String externalIp = ipPortExternal.getIpAddress();
2029                 int externalPort = ipPortExternal.getPortNum();
2030
2031                 //Build the flow for the inbound NAPT table
2032                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2033                     String.valueOf(routerId), externalIp, externalPort);
2034                 FlowEntity inboundNaptFlowEntity =
2035                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2036
2037                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2038                         + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2039                 mdsalManager.removeFlowToTx(inboundNaptFlowEntity, removeFlowInvTx);
2040             }
2041         }
2042     }
2043
2044     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2045                                                     @Nonnull Collection<String> externalIps,
2046                                                     WriteTransaction writeFlowInvTx) {
2047         long extVpnId = NatConstants.INVALID_ID;
2048         if (networkId != null) {
2049             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2050             if (vpnUuid != null) {
2051                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2052             } else {
2053                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2054             }
2055         } else {
2056             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2057             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2058         }
2059         if (extVpnId == NatConstants.INVALID_ID) {
2060             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2061             extVpnId = routerId;
2062         }
2063         for (String ip : externalIps) {
2064             String extIp = removeMaskFromIp(ip);
2065             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2066             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2067                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2068                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2069             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2070             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2071         }
2072     }
2073
2074     private String removeMaskFromIp(String ip) {
2075         if (ip != null && !ip.trim().isEmpty()) {
2076             return ip.split("/")[0];
2077         }
2078         return ip;
2079     }
2080
2081     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2082                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2083                                                            WriteTransaction writeFlowInvTx) {
2084         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2085         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2086
2087         //Remove the NAPT PFIB TABLE entry
2088         long vpnId = -1;
2089         if (vpnName != null) {
2090             // ie called from disassociate vpn case
2091             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2092                     + "with vpnName {}", vpnName);
2093             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2094             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2095                     vpnId);
2096         }
2097
2098         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2099                 networkId, routerName, dpnId)) {
2100             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2101             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2102             FlowEntity natPfibVpnFlowEntity =
2103                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2104             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2105                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2106             mdsalManager.removeFlowToTx(natPfibVpnFlowEntity, writeFlowInvTx);
2107
2108             // Remove IP-PORT active NAPT entries and release port from IdManager
2109             // For the router ID get the internal IP , internal port and the corresponding
2110             // external IP and external Port.
2111             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2112             if (ipPortMapping == null) {
2113                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2114                 return;
2115             }
2116             List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2117             for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2118                 List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2119                 for (IpPortMap ipPortMap : ipPortMaps) {
2120                     String ipPortInternal = ipPortMap.getIpPortInternal();
2121                     String[] ipPortParts = ipPortInternal.split(":");
2122                     if (ipPortParts.length != 2) {
2123                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2124                                 + "and port");
2125                         return;
2126                     }
2127                     String internalIp = ipPortParts[0];
2128                     String internalPort = ipPortParts[1];
2129
2130                     //Build the flow for the outbound NAPT table
2131                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2132                             + NatConstants.COLON_SEPARATOR + internalPort);
2133                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2134                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort));
2135                     FlowEntity outboundNaptFlowEntity =
2136                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2137
2138                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2139                             + "active switch with the DPN ID {} and router ID {}",
2140                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2141                     mdsalManager.removeFlowToTx(outboundNaptFlowEntity, writeFlowInvTx);
2142
2143                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2144                     String externalIp = ipPortExternal.getIpAddress();
2145                     int externalPort = ipPortExternal.getPortNum();
2146
2147                     //Build the flow for the inbound NAPT table
2148                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2149                         String.valueOf(routerId), externalIp, externalPort);
2150                     FlowEntity inboundNaptFlowEntity =
2151                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2152
2153                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2154                             + "active active switch with the DPN ID {} and router ID {}",
2155                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2156                     mdsalManager.removeFlowToTx(inboundNaptFlowEntity, writeFlowInvTx);
2157
2158                     // Finally release port from idmanager
2159                     String internalIpPort = internalIp + ":" + internalPort;
2160                     naptManager.removePortFromPool(internalIpPort, externalIp);
2161
2162                     //Remove sessions from models
2163                     naptManager.removeIpPortMappingForRouterID(routerId);
2164                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2165                 }
2166             }
2167         } else {
2168             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2169         }
2170     }
2171
2172     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2173             BigInteger naptSwitchDpnId, WriteTransaction removeFlowInvTx) {
2174         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2175
2176         // Remove the flows from the other switches which points to the primary and secondary switches
2177         // for the flows related the router ID.
2178         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2179         if (allSwitchList.isEmpty()) {
2180             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2181             return;
2182         }
2183         for (BigInteger dpnId : allSwitchList) {
2184             if (!naptSwitchDpnId.equals(dpnId)) {
2185                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2186
2187                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2188                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2189                 FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2190
2191                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2192                         + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2193                 mdsalManager.removeFlowToTx(preSnatFlowEntity, removeFlowInvTx);
2194
2195                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2196                 long groupId = createGroupId(getGroupIdKey(routerName));
2197                 List<BucketInfo> listBucketInfo = new ArrayList<>();
2198                 GroupEntity preSnatGroupEntity =
2199                     MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, listBucketInfo);
2200
2201                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2202                         + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2203                 mdsalManager.removeGroup(preSnatGroupEntity);
2204
2205             }
2206         }
2207     }
2208
2209     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2210                                          @Nonnull Collection<String> externalIps, String vpnName,
2211                                          String extGwMacAddress, WriteTransaction removeFlowInvTx) {
2212         //Withdraw the corresponding routes from the BGP.
2213         //Get the network ID using the router ID.
2214         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2215                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2216         if (networkUuid == null) {
2217             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2218             return;
2219         }
2220
2221         if (externalIps.isEmpty()) {
2222             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2223             return;
2224         }
2225
2226         if (vpnName == null) {
2227             //Get the VPN Name using the network ID
2228             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2229             if (vpnName == null) {
2230                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2231                     networkUuid, routerId);
2232                 return;
2233             }
2234         }
2235         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2236
2237         //Remove custom FIB routes
2238         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2239         for (String extIp : externalIps) {
2240             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, removeFlowInvTx);
2241         }
2242     }
2243
2244     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2245                                             final Uuid networkUuid, String extGwMacAddress,
2246                                             WriteTransaction removeFlowInvTx) {
2247         clearBgpRoutes(extIp, vpnName);
2248         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2249                 removeFlowInvTx);
2250     }
2251
2252     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp,
2253                                              final String vpnName, Uuid extNetworkId, long tempLabel,
2254                                              String gwMacAddress, boolean switchOver,
2255                                              WriteTransaction removeFlowInvTx) {
2256         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2257         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2258         if (routerName == null) {
2259             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2260             return;
2261         }
2262         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2263         if (extNwProvType == null) {
2264             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2265             return;
2266         }
2267         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2268          * external network provided type is VxLAN
2269          */
2270         if (extNwProvType == ProviderTypes.VXLAN) {
2271             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress,
2272                     removeFlowInvTx);
2273             return;
2274         }
2275         if (tempLabel < 0) {
2276             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2277                     extIp, routerId);
2278             return;
2279         }
2280
2281         final long label = tempLabel;
2282         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2283         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
2284                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2285                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2286         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2287
2288         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2289         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2290         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2291             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2292             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2293         }
2294         if (!switchOver) {
2295             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2296                     Futures.transformAsync(future, result -> {
2297                                 //Release label
2298                         if (result.isSuccessful()) {
2299                             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2300                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2301                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2302                             return vpnService.removeVpnLabel(labelInput);
2303                         } else {
2304                             String errMsg =
2305                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2306                                             + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2307                             LOG.error(errMsg);
2308                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2309                         }
2310                     }, MoreExecutors.directExecutor());
2311
2312             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2313
2314                 @Override
2315                 public void onFailure(@Nonnull Throwable error) {
2316                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2317                         + "got external ip {}", label, extIp, error);
2318                 }
2319
2320                 @Override
2321                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2322                     if (result.isSuccessful()) {
2323                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2324                             + "from VPN {}", externalIp, vpnName);
2325                     } else {
2326                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2327                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2328                     }
2329                 }
2330             }, MoreExecutors.directExecutor());
2331         } else {
2332             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2333                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2334         }
2335     }
2336
2337     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2338                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2339                                            WriteTransaction removeFlowInvTx) {
2340         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2341         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2342         if (routerName == null) {
2343             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2344             return;
2345         }
2346         //Get the external network provider type from networkId
2347         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2348         if (extNwProvType == null) {
2349             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2350                     networkUuid);
2351             return;
2352         }
2353         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2354          *  external network provided type is VxLAN
2355          */
2356         if (extNwProvType == ProviderTypes.VXLAN) {
2357             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress,
2358                     removeFlowInvTx);
2359             return;
2360         }
2361         //Get IPMaps from the DB for the router ID
2362         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2363         if (dbIpMaps.isEmpty()) {
2364             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2365             return;
2366         }
2367
2368         long tempLabel = NatConstants.INVALID_ID;
2369         for (IpMap dbIpMap : dbIpMaps) {
2370             String dbExternalIp = dbIpMap.getExternalIp();
2371             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2372             //Select the IPMap, whose external IP is the IP for which FIB is installed
2373             if (extIp.equals(dbExternalIp)) {
2374                 tempLabel = dbIpMap.getLabel();
2375                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2376                     tempLabel, dbExternalIp, routerId);
2377                 break;
2378             }
2379         }
2380         if (tempLabel == NatConstants.INVALID_ID) {
2381             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2382                     extIp, routerId);
2383             return;
2384         }
2385
2386         final long label = tempLabel;
2387         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2388         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2389                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2390                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2391         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2392
2393         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2394         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2395         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2396             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2397             NatUtil.removePreDnatToSnatTableEntry(mdsalManager, dpnId, removeFlowInvTx);
2398         }
2399         if (!switchOver) {
2400             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2401                     Futures.transformAsync(future, result -> {
2402                         //Release label
2403                         if (result.isSuccessful()) {
2404                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2405                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2406                             return vpnService.removeVpnLabel(labelInput);
2407                         } else {
2408                             String errMsg =
2409                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2410                                             + "prefix %s Failed - %s",
2411                                             dpnId, externalIp, result.getErrors());
2412                             LOG.error(errMsg);
2413                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2414                         }
2415                     }, MoreExecutors.directExecutor());
2416
2417             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2418
2419                 @Override
2420                 public void onFailure(@Nonnull Throwable error) {
2421                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2422                 }
2423
2424                 @Override
2425                 public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
2426                     if (result.isSuccessful()) {
2427                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2428                             + "from VPN {}", externalIp, vpnName);
2429                     } else {
2430                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2431                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2432                     }
2433                 }
2434             }, MoreExecutors.directExecutor());
2435         } else {
2436             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2437                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2438         }
2439     }
2440
2441     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2442                                                List<String> externalIps, String vpnName, String extGwMacAddress,
2443                                                WriteTransaction writeFlowInvTx) {
2444         //Withdraw the corresponding routes from the BGP.
2445         //Get the network ID using the router ID.
2446         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2447                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2448         if (networkUuid == null) {
2449             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2450             return;
2451         }
2452
2453         if (externalIps == null || externalIps.isEmpty()) {
2454             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2455             return;
2456         }
2457
2458         if (vpnName == null) {
2459             //Get the VPN Name using the network ID
2460             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2461             if (vpnName == null) {
2462                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2463                     networkUuid, routerId);
2464                 return;
2465             }
2466         }
2467         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2468
2469         //Remove custom FIB routes
2470         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2471         for (String extIp : externalIps) {
2472             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2473                     writeFlowInvTx);
2474         }
2475     }
2476
2477     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2478         //Inform BGP about the route removal
2479         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2480         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2481         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2482     }
2483
2484     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2485         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2486         List<MatchInfo> mkMatches = new ArrayList<>();
2487         // Matching metadata
2488         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
2489         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
2490             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""),
2491             5, String.format("%s:%d", "TST Flow Entry ", serviceId), 0, 0,
2492             COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, null);
2493         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2494         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2495     }
2496
2497     private void removeLFibTableEntry(BigInteger dpnId, long serviceId, WriteTransaction writeFlowInvTx) {
2498         List<MatchInfo> matches = new ArrayList<>();
2499         matches.add(MatchEthernetType.MPLS_UNICAST);
2500         matches.add(new MatchMplsLabel(serviceId));
2501
2502         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2503
2504         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2505
2506         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
2507             10, flowRef, 0, 0,
2508             COOKIE_VM_LFIB_TABLE, matches, null);
2509
2510         mdsalManager.removeFlowToTx(dpnId, flowEntity, writeFlowInvTx);
2511
2512         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2513     }
2514
2515     /**
2516      * router association to vpn.
2517      *
2518      * @param routerName - Name of router
2519      * @param routerId - router id
2520      * @param bgpVpnName BGP VPN name
2521      */
2522     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2523             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2524         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2525         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2526             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2527
2528             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2529
2530             if (bgpVpnId != NatConstants.INVALID_ID) {
2531                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2532                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2533                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2534                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2535                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2536                     getRoutersIdentifier(bgpVpnId), rtrs);
2537
2538                 // Get the allocated Primary NAPT Switch for this router
2539                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2540
2541                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2542                         routerId, bgpVpnId);
2543                 addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, true, writeFlowInvTx);
2544
2545                 // Get the group ID
2546                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2547                 createGroupId(getGroupIdKey(routerName));
2548                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2549                         extNwProvType);
2550             }
2551         }
2552     }
2553
2554     /**
2555      * router disassociation from vpn.
2556      *
2557      * @param routerName - Name of router
2558      * @param routerId - router id
2559      * @param bgpVpnName BGP VPN name
2560      */
2561     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2562             WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2563         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2564         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2565             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2566             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2567
2568             // Get the allocated Primary NAPT Switch for this router
2569             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2570
2571             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2572             addOrDelDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID,
2573                     true, writeFlowInvTx);
2574
2575             // Get the group ID
2576             createGroupId(getGroupIdKey(routerName));
2577             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2578             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2579                     writeFlowInvTx, extNwProvType);
2580         }
2581     }
2582
2583     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2584         InstanceIdentifier<Routers> routerInstanceIndentifier =
2585             InstanceIdentifier.builder(ExtRouters.class)
2586                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2587         Optional<Routers> routerData = read(dataBroker, LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2588         return routerData.isPresent() && routerData.get().isEnableSnat();
2589     }
2590
2591     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2592                                              long routerId, boolean isSnatCfgd, WriteTransaction writeFlowInvTx,
2593                                              ProviderTypes extNwProvType) {
2594         long changedVpnId = bgpVpnId;
2595         String idType = "BGP VPN";
2596         if (bgpVpnId == NatConstants.INVALID_ID) {
2597             changedVpnId = routerId;
2598             idType = "router";
2599         }
2600
2601         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2602         if (switches.isEmpty()) {
2603             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2604             return;
2605         }
2606         for (BigInteger dpnId : switches) {
2607             // Update the BGP VPN ID in the SNAT miss entry to group
2608             if (!dpnId.equals(primarySwitchId)) {
2609                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2610                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2611                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2612                 long groupId = createGroupId(getGroupIdKey(routerName));
2613                 if (!isSnatCfgd) {
2614                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2615                 }
2616
2617                 LOG.debug(
2618                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2619                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2620                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2621                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2622             } else {
2623                 LOG.debug(
2624                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2625                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2626                 FlowEntity flowEntity =
2627                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2628                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2629
2630                 LOG.debug(
2631                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2632                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2633                         idType, changedVpnId, primarySwitchId);
2634                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2635                         changedVpnId, writeFlowInvTx, extNwProvType);
2636
2637                 LOG.debug(
2638                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2639                                 + "which punts the packet to the controller in the Primary switch {}",
2640                         idType, changedVpnId, primarySwitchId);
2641                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2642
2643                 LOG.debug(
2644                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2645                                 + " outgoing packet to FIB Table in the Primary switch {}",
2646                         idType, changedVpnId, primarySwitchId);
2647                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, writeFlowInvTx);
2648
2649                 LOG.debug(
2650                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2651                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2652                                 + " {}", idType, changedVpnId, primarySwitchId);
2653                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2654
2655                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2656                         primarySwitchId);
2657                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2658                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2659                 if (vpnId != NatConstants.INVALID_ID) {
2660                     installNaptPfibEntry(primarySwitchId, vpnId, writeFlowInvTx);
2661                 }
2662             }
2663         }
2664     }
2665
2666     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2667         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2668         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2669         if (ipPortMapping == null) {
2670             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2671             return;
2672         }
2673         // Get the External Gateway MAC Address
2674         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2675         if (extGwMacAddress != null) {
2676             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2677                     extGwMacAddress, routerId);
2678         } else {
2679             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2680                     routerId);
2681             return;
2682         }
2683         List<IntextIpProtocolType> intextIpProtocolTypes = ipPortMapping.getIntextIpProtocolType();
2684         for (IntextIpProtocolType intextIpProtocolType : intextIpProtocolTypes) {
2685             List<IpPortMap> ipPortMaps = intextIpProtocolType.getIpPortMap();
2686             for (IpPortMap ipPortMap : ipPortMaps) {
2687                 String ipPortInternal = ipPortMap.getIpPortInternal();
2688                 String[] ipPortParts = ipPortInternal.split(":");
2689                 if (ipPortParts.length != 2) {
2690                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2691                     return;
2692                 }
2693                 String internalIp = ipPortParts[0];
2694                 String internalPort = ipPortParts[1];
2695                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2696                         internalIp, internalPort);
2697                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2698                 NAPTEntryEvent.Protocol protocol;
2699                 switch (protocolTypes) {
2700                     case TCP:
2701                         protocol = NAPTEntryEvent.Protocol.TCP;
2702                         break;
2703                     case UDP:
2704                         protocol = NAPTEntryEvent.Protocol.UDP;
2705                         break;
2706                     default:
2707                         protocol = NAPTEntryEvent.Protocol.TCP;
2708                 }
2709                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2710                 SessionAddress externalAddress =
2711                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2712                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2713                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2714                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2715                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2716                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2717             }
2718         }
2719     }
2720
2721     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2722                                                           long changedVpnId) {
2723
2724         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2725             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2726         List<MatchInfo> matches = new ArrayList<>();
2727         matches.add(MatchEthernetType.IPV4);
2728         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2729
2730         List<ActionInfo> actionsInfo = new ArrayList<>();
2731         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, elanManager, idManager, changedVpnId,
2732                 routerName);
2733         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2734         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2735                 actionsInfo);
2736         actionsInfo.add(new ActionGroup(groupId));
2737         List<InstructionInfo> instructions = new ArrayList<>();
2738         instructions.add(new InstructionApplyActions(actionsInfo));
2739         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2740         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2741             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2742             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2743
2744         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2745         return flowEntity;
2746     }
2747
2748     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2749                                                                         long changedVpnId) {
2750
2751         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2752                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2753         List<MatchInfo> matches = new ArrayList<>();
2754         matches.add(MatchEthernetType.IPV4);
2755         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2756
2757         List<InstructionInfo> instructions = new ArrayList<>();
2758         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2759
2760         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2761         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2762             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2763             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2764
2765         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2766         return flowEntity;
2767     }
2768
2769     // TODO : Replace this with ITM Rpc once its available with full functionality
2770     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2771             long routerId, long changedVpnId, WriteTransaction writeFlowInvTx, ProviderTypes extNwProvType) {
2772         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2773             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2774         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2775                 extNwProvType);
2776         mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
2777     }
2778
2779     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2780             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2781         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2782             dpId, routerName, changedVpnId);
2783         List<MatchInfo> matches = new ArrayList<>();
2784         matches.add(MatchEthernetType.IPV4);
2785
2786         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2787         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2788             tunnelId = NatOverVxlanUtil.getRouterVni(idManager, routerName, changedVpnId);
2789         }
2790         matches.add(new MatchTunnelId(tunnelId));
2791
2792         List<InstructionInfo> instructions = new ArrayList<>();
2793         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2794             MetaDataUtil.METADATA_MASK_VRFID));
2795         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2796         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2797         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2798         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2799             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2800             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2801         return flowEntity;
2802     }
2803
2804     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2805                                                  WriteTransaction writeFlowInvTx) {
2806         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2807             dpnId, routerId, changedVpnId);
2808         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2809                 NwConstants.IP_PROT_TCP);
2810         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2811         mdsalManager.addFlowToTx(tcpFlowEntity, writeFlowInvTx);
2812
2813         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2814                 NwConstants.IP_PROT_UDP);
2815         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2816         mdsalManager.addFlowToTx(udpFlowEntity, writeFlowInvTx);
2817
2818         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2819         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2820         mdsalManager.addFlowToTx(icmpDropFlow, writeFlowInvTx);
2821     }
2822
2823     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2824                                                            long changedVpnId, int protocol) {
2825         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2826             dpId, routerId, changedVpnId);
2827         BigInteger cookie = getCookieOutboundFlow(routerId);
2828         List<MatchInfo> matches = new ArrayList<>();
2829         matches.add(MatchEthernetType.IPV4);
2830         matches.add(new MatchIpProtocol((short)protocol));
2831         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2832
2833         List<InstructionInfo> instructions = new ArrayList<>();
2834         List<ActionInfo> actionsInfos = new ArrayList<>();
2835         actionsInfos.add(new ActionPuntToController());
2836         if (snatPuntTimeout != 0) {
2837             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2838         }
2839         instructions.add(new InstructionApplyActions(actionsInfos));
2840
2841         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2842         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2843             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2844         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2845         return flowEntity;
2846     }
2847
2848     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2849                                                WriteTransaction writeFlowInvTx) {
2850         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2851             dpnId, segmentId, changedVpnId);
2852         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2853         mdsalManager.addFlowToTx(naptPfibFlowEntity, writeFlowInvTx);
2854     }
2855
2856     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2857
2858         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2859             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2860         List<MatchInfo> matches = new ArrayList<>();
2861         matches.add(MatchEthernetType.IPV4);
2862         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2863
2864         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2865         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2866         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2867         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2868         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2869
2870         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2871         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2872             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2873             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2874         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2875         return flowEntity;
2876     }
2877
2878     @Override
2879     protected ExternalRoutersListener getDataTreeChangeListener() {
2880         return ExternalRoutersListener.this;
2881     }
2882
2883     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2884                                                             WriteTransaction writeFlowInvTx) {
2885         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2886                 routerName);
2887         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2888             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2889             if (subnetVpnId != -1) {
2890                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2891                     + "and vpnId {}", dpnId, subnetVpnId);
2892                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2893             }
2894         }
2895     }
2896 }