natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/ExternalRoutersListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.net.Inet6Address;
  20 import java.net.InetAddress;
  21 import java.net.UnknownHostException;
  22 import java.util.ArrayList;
  23 import java.util.Arrays;
  24 import java.util.Collection;
  25 import java.util.Collections;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.Future;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  41 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  42 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.Datastore.Configuration;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.mdsalutil.ActionInfo;
  51 import org.opendaylight.genius.mdsalutil.BucketInfo;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.GroupEntity;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.NwConstants.NxmOfFieldType;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionLearn;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromField;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionLearn.MatchFromValue;
  65 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
  66 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  67 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  68 import org.opendaylight.genius.mdsalutil.actions.ActionPuntToController;
  69 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  70 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  71 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  72 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  73 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  74 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  75 import org.opendaylight.genius.mdsalutil.matches.MatchIpProtocol;
  76 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  77 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  78 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  79 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  80 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  81 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  82 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  83 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  84 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  85 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  86 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  87 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalIpsCounter;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.IntextIpPortMap;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProtocolTypes;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.RouterIdName;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.RoutersKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.ExternalCounters;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.ips.counter.external.counters.ExternalIpCounter;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMap;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.map.ip.mapping.IpMapKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMapping;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.IpPortMappingKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.IntextIpProtocolType;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.IpPortMap;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.intext.ip.port.map.ip.port.mapping.intext.ip.protocol.type.ip.port.map.IpPortExternal;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIds;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.router.id.name.RouterIdsKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
 143 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
 144 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
 145 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
 146 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
 147 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 148 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 149 import org.opendaylight.yangtools.yang.common.RpcResult;
 150 import org.slf4j.Logger;
 151 import org.slf4j.LoggerFactory;
 152
 153
 154 @Singleton
 155 public class ExternalRoutersListener extends AsyncDataTreeChangeListenerBase<Routers, ExternalRoutersListener> {
 156     private static final Logger LOG = LoggerFactory.getLogger(ExternalRoutersListener.class);
 157
 158     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 159     private static final BigInteger COOKIE_VM_LFIB_TABLE = new BigInteger("8000022", 16);
 160
 161     private final DataBroker dataBroker;
 162     private final ManagedNewTransactionRunner txRunner;
 163     private final IMdsalApiManager mdsalManager;
 164     private final ItmRpcService itmManager;
 165     private final OdlInterfaceRpcService odlInterfaceRpcService;
 166     private final IdManagerService idManager;
 167     private final NaptManager naptManager;
 168     private final NAPTSwitchSelector naptSwitchSelector;
 169     private final IBgpManager bgpManager;
 170     private final VpnRpcService vpnService;
 171     private final FibRpcService fibService;
 172     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
 173     private final NaptEventHandler naptEventHandler;
 174     private final NaptPacketInHandler naptPacketInHandler;
 175     private final IFibManager fibManager;
 176     private final IVpnManager vpnManager;
 177     private final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer;
 178     private final NatMode natMode;
 179     private final IElanService elanManager;
 180     private final JobCoordinator coordinator;
 181     private final IInterfaceManager interfaceManager;
 182     private final NatOverVxlanUtil natOverVxlanUtil;
 183     private final int snatPuntTimeout;
 184
 185     @Inject
 186     public ExternalRoutersListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 187                                    final ItmRpcService itmManager,
 188                                    final OdlInterfaceRpcService odlInterfaceRpcService,
 189                                    final IdManagerService idManager,
 190                                    final NaptManager naptManager,
 191                                    final NAPTSwitchSelector naptSwitchSelector,
 192                                    final IBgpManager bgpManager,
 193                                    final VpnRpcService vpnService,
 194                                    final FibRpcService fibService,
 195                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
 196                                    final NaptEventHandler naptEventHandler,
 197                                    final NaptPacketInHandler naptPacketInHandler,
 198                                    final IFibManager fibManager,
 199                                    final IVpnManager vpnManager,
 200                                    final EvpnSnatFlowProgrammer evpnSnatFlowProgrammer,
 201                                    final NatserviceConfig config,
 202                                    final IElanService elanManager,
 203                                    final JobCoordinator coordinator,
 204                                    final NatOverVxlanUtil natOverVxlanUtil,
 205                                    final IInterfaceManager interfaceManager) {
 206         super(Routers.class, ExternalRoutersListener.class);
 207         this.dataBroker = dataBroker;
 208         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 209         this.mdsalManager = mdsalManager;
 210         this.itmManager = itmManager;
 211         this.odlInterfaceRpcService = odlInterfaceRpcService;
 212         this.idManager = idManager;
 213         this.naptManager = naptManager;
 214         this.naptSwitchSelector = naptSwitchSelector;
 215         this.bgpManager = bgpManager;
 216         this.vpnService = vpnService;
 217         this.fibService = fibService;
 218         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
 219         this.naptEventHandler = naptEventHandler;
 220         this.naptPacketInHandler = naptPacketInHandler;
 221         this.fibManager = fibManager;
 222         this.vpnManager = vpnManager;
 223         this.evpnSnatFlowProgrammer = evpnSnatFlowProgrammer;
 224         this.elanManager = elanManager;
 225         this.coordinator = coordinator;
 226         this.interfaceManager = interfaceManager;
 227         this.natOverVxlanUtil = natOverVxlanUtil;
 228         if (config != null) {
 229             this.natMode = config.getNatMode();
 230             this.snatPuntTimeout = config.getSnatPuntTimeout().intValue();
 231         } else {
 232             this.natMode = NatMode.Controller;
 233             this.snatPuntTimeout = 0;
 234         }
 235     }
 236
 237     @Override
 238     @PostConstruct
 239     public void init() {
 240         LOG.info("{} init", getClass().getSimpleName());
 241         // This class handles ExternalRouters for Controller SNAT mode.
 242         // For Conntrack SNAT mode, its handled in SnatExternalRoutersListener.java
 243         if (natMode == NatMode.Controller) {
 244             registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 245             NatUtil.createGroupIdPool(idManager);
 246         }
 247     }
 248
 249     @Override
 250     protected InstanceIdentifier<Routers> getWildCardPath() {
 251         return InstanceIdentifier.create(ExtRouters.class).child(Routers.class);
 252     }
 253
 254     @Override
 255     // TODO Clean up the exception handling
 256     @SuppressWarnings("checkstyle:IllegalCatch")
 257     protected void add(InstanceIdentifier<Routers> identifier, Routers routers) {
 258         // Populate the router-id-name container
 259         String routerName = routers.getRouterName();
 260         LOG.info("add : external router event for {}", routerName);
 261         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 262         NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 263         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
 264         try {
 265             if (routers.isEnableSnat()) {
 266                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routers.key(),
 267                     () -> Collections.singletonList(
 268                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 269                             LOG.info("add : Installing NAT default route on all dpns part of router {}", routerName);
 270                             long bgpVpnId = NatConstants.INVALID_ID;
 271                             if (bgpVpnUuid != null) {
 272                                 bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
 273                             }
 274                             addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, true, confTx);
 275                             // Allocate Primary Napt Switch for this router
 276                             BigInteger primarySwitchId = getPrimaryNaptSwitch(routerName);
 277                             if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 278                                 handleEnableSnat(routers, routerId, primarySwitchId, bgpVpnId, confTx);
 279                             }
 280                         }
 281                     )), NatConstants.NAT_DJC_MAX_RETRIES);
 282             } else {
 283                 LOG.info("add : SNAT is disabled for external router {} ", routerName);
 284             }
 285         } catch (Exception ex) {
 286             LOG.error("add : Exception while Installing NAT flows on all dpns as part of router {}",
 287                     routerName, ex);
 288         }
 289     }
 290
 291     public void handleEnableSnat(Routers routers, long routerId, BigInteger primarySwitchId, long bgpVpnId,
 292                                  TypedWriteTransaction<Configuration> confTx) {
 293         String routerName = routers.getRouterName();
 294         LOG.info("handleEnableSnat : Handling SNAT for router {}", routerName);
 295
 296         naptManager.initialiseExternalCounter(routers, routerId);
 297         subnetRegisterMapping(routers, routerId);
 298
 299         LOG.debug("handleEnableSnat:About to create and install outbound miss entry in Primary Switch {} for router {}",
 300             primarySwitchId, routerName);
 301
 302         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
 303                 routers.getNetworkId());
 304         if (extNwProvType == null) {
 305             LOG.error("handleEnableSnat : External Network Provider Type missing");
 306             return;
 307         }
 308
 309         if (bgpVpnId != NatConstants.INVALID_ID) {
 310             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, false, confTx,
 311                     extNwProvType);
 312         } else {
 313             // write metadata and punt
 314             installOutboundMissEntry(routerName, routerId, primarySwitchId, confTx);
 315             handlePrimaryNaptSwitch(primarySwitchId, routerName, routerId, confTx);
 316             // Now install entries in SNAT tables to point to Primary for each router
 317             List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 318             for (BigInteger dpnId : switches) {
 319                 // Handle switches and NAPT switches separately
 320                 if (!dpnId.equals(primarySwitchId)) {
 321                     LOG.debug("handleEnableSnat : Handle Ordinary switch");
 322                     handleSwitches(dpnId, routerName, routerId, primarySwitchId);
 323                 }
 324             }
 325         }
 326
 327         Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 328         if (externalIps.isEmpty()) {
 329             LOG.error("handleEnableSnat : Internal External mapping not found for router {}", routerName);
 330             return;
 331         } else {
 332             for (String externalIpAddrPrefix : externalIps) {
 333                 LOG.debug("handleEnableSnat : Calling handleSnatReverseTraffic for primarySwitchId {}, "
 334                     + "routerName {} and externalIpAddPrefix {}", primarySwitchId, routerName, externalIpAddrPrefix);
 335                 handleSnatReverseTraffic(confTx, primarySwitchId, routers, routerId, routerName, externalIpAddrPrefix
 336                 );
 337             }
 338         }
 339         LOG.debug("handleEnableSnat : Exit");
 340     }
 341
 342     private BigInteger getPrimaryNaptSwitch(String routerName) {
 343         // Allocate Primary Napt Switch for this router
 344         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 345         if (primarySwitchId != null && !primarySwitchId.equals(BigInteger.ZERO)) {
 346             LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch with DPN ID {} is already elected for router {}",
 347                 primarySwitchId, routerName);
 348             return primarySwitchId;
 349         }
 350         // Allocated an id from VNI pool for the Router.
 351         natOverVxlanUtil.getRouterVni(routerName, NatConstants.INVALID_ID);
 352         primarySwitchId = naptSwitchSelector.selectNewNAPTSwitch(routerName);
 353         LOG.debug("getPrimaryNaptSwitch : Primary NAPT switch DPN ID {}", primarySwitchId);
 354
 355         return primarySwitchId;
 356     }
 357
 358     protected void installNaptPfibExternalOutputFlow(String routerName, Long routerId, BigInteger dpnId,
 359                                                      TypedWriteTransaction<Configuration> confTx) {
 360         Long extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
 361         if (extVpnId == NatConstants.INVALID_ID) {
 362             LOG.error("installNaptPfibExternalOutputFlow - not found extVpnId for router {}", routerId);
 363             extVpnId = routerId;
 364         }
 365         List<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerName);
 366         if (externalIps.isEmpty()) {
 367             LOG.error("installNaptPfibExternalOutputFlow - empty external Ips list for dpnId {} extVpnId {}",
 368                 dpnId, extVpnId);
 369             return;
 370         }
 371         for (String ip : externalIps) {
 372             Uuid subnetId = getSubnetIdForFixedIp(ip);
 373             if (subnetId != null) {
 374                 long subnetVpnId = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 375                 if (subnetVpnId != NatConstants.INVALID_ID) {
 376                     extVpnId = subnetVpnId;
 377                 }
 378                 LOG.debug("installNaptPfibExternalOutputFlow - dpnId {} extVpnId {} subnetId {}",
 379                     dpnId, extVpnId, subnetId);
 380                 FlowEntity postNaptFlowEntity = buildNaptPfibFlowEntity(dpnId, extVpnId);
 381                 mdsalManager.addFlow(confTx, postNaptFlowEntity);
 382             }
 383         }
 384     }
 385
 386     @Nullable
 387     private Uuid getSubnetIdForFixedIp(String ip) {
 388         if (ip != null) {
 389             IpAddress externalIpv4Address = new IpAddress(new Ipv4Address(ip));
 390             Port port = NatUtil.getNeutronPortForRouterGetewayIp(dataBroker, externalIpv4Address);
 391             return NatUtil.getSubnetIdForFloatingIp(port, externalIpv4Address);
 392         }
 393         LOG.error("getSubnetIdForFixedIp : ip is null");
 394         return null;
 395     }
 396
 397     protected void subnetRegisterMapping(Routers routerEntry, Long segmentId) {
 398         LOG.debug("subnetRegisterMapping : Fetching values from extRouters model");
 399         List<String> externalIps = NatUtil.getIpsListFromExternalIps(routerEntry.getExternalIps());
 400         int counter = 0;
 401         int extIpCounter = externalIps.size();
 402         LOG.debug("subnetRegisterMapping : counter values before looping counter {} and extIpCounter {}",
 403                 counter, extIpCounter);
 404         @Nullable List<Uuid> subnetIds = routerEntry.getSubnetIds();
 405         if (subnetIds == null) {
 406             return;
 407         }
 408         for (Uuid subnet : subnetIds) {
 409             LOG.debug("subnetRegisterMapping : Looping internal subnets for subnet {}", subnet);
 410             InstanceIdentifier<Subnetmap> subnetmapId = InstanceIdentifier
 411                 .builder(Subnetmaps.class)
 412                 .child(Subnetmap.class, new SubnetmapKey(subnet))
 413                 .build();
 414             Optional<Subnetmap> sn;
 415             try {
 416                 sn = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 417                                 LogicalDatastoreType.CONFIGURATION, subnetmapId);
 418             } catch (ReadFailedException e) {
 419                 LOG.error("Failed to read SubnetMap for  subnetmap Id {}", subnetmapId, e);
 420                 sn = Optional.absent();
 421             }
 422             if (sn.isPresent()) {
 423                 // subnets
 424                 Subnetmap subnetmapEntry = sn.get();
 425                 String subnetString = subnetmapEntry.getSubnetIp();
 426                 String[] subnetSplit = subnetString.split("/");
 427                 String subnetIp = subnetSplit[0];
 428                 try {
 429                     InetAddress address = InetAddress.getByName(subnetIp);
 430                     if (address instanceof Inet6Address) {
 431                         LOG.debug("subnetRegisterMapping : Skipping ipv6 subnet {} for the router {} with ipv6 address "
 432                                 + "{} ", subnet, routerEntry.getRouterName(), address);
 433                         continue;
 434                     }
 435                 } catch (UnknownHostException e) {
 436                     LOG.error("subnetRegisterMapping : Invalid ip address {}", subnetIp, e);
 437                     return;
 438                 }
 439                 String subnetPrefix = "0";
 440                 if (subnetSplit.length == 2) {
 441                     subnetPrefix = subnetSplit[1];
 442                 }
 443                 IPAddress subnetAddr = new IPAddress(subnetIp, Integer.parseInt(subnetPrefix));
 444                 LOG.debug("subnetRegisterMapping : subnetAddr is {} and subnetPrefix is {}",
 445                     subnetAddr.getIpAddress(), subnetAddr.getPrefixLength());
 446                 //externalIps
 447                 LOG.debug("subnetRegisterMapping : counter values counter {} and extIpCounter {}",
 448                         counter, extIpCounter);
 449                 if (extIpCounter != 0) {
 450                     if (counter < extIpCounter) {
 451                         String[] ipSplit = externalIps.get(counter).split("/");
 452                         String externalIp = ipSplit[0];
 453                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 454                         if (ipSplit.length == 2) {
 455                             extPrefix = ipSplit[1];
 456                         }
 457                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 458                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 459                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 460                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 461                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 462                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix, externalIp, extPrefix);
 463                     } else {
 464                         counter = 0;    //Reset the counter which runs on externalIps for round-robbin effect
 465                         LOG.debug("subnetRegisterMapping : Counter on externalIps got reset");
 466                         String[] ipSplit = externalIps.get(counter).split("/");
 467                         String externalIp = ipSplit[0];
 468                         String extPrefix = Short.toString(NatConstants.DEFAULT_PREFIX);
 469                         if (ipSplit.length == 2) {
 470                             extPrefix = ipSplit[1];
 471                         }
 472                         IPAddress externalIpAddr = new IPAddress(externalIp, Integer.parseInt(extPrefix));
 473                         LOG.debug("subnetRegisterMapping : externalIp is {} and extPrefix  is {}",
 474                             externalIpAddr.getIpAddress(), externalIpAddr.getPrefixLength());
 475                         naptManager.registerMapping(segmentId, subnetAddr, externalIpAddr);
 476                         LOG.debug("subnetRegisterMapping : Called registerMapping for subnetIp {}, prefix {}, "
 477                                 + "externalIp {}. prefix {}", subnetIp, subnetPrefix,
 478                             externalIp, extPrefix);
 479                     }
 480                 }
 481                 counter++;
 482                 LOG.debug("subnetRegisterMapping : Counter on externalIps incremented to {}", counter);
 483             } else {
 484                 LOG.warn("subnetRegisterMapping : No internal subnets present in extRouters Model");
 485             }
 486         }
 487     }
 488
 489     private void addOrDelDefFibRouteToSNAT(String routerName, long routerId, long bgpVpnId,
 490             Uuid bgpVpnUuid, boolean create, TypedReadWriteTransaction<Configuration> confTx)
 491             throws ExecutionException, InterruptedException {
 492         //Check if BGP VPN exists. If exists then invoke the new method.
 493         if (bgpVpnId != NatConstants.INVALID_ID) {
 494             if (bgpVpnUuid != null) {
 495                 String bgpVpnName = bgpVpnUuid.getValue();
 496                 LOG.debug("Populate the router-id-name container with the mapping BGP VPN-ID {} -> BGP VPN-NAME {}",
 497                     bgpVpnId, bgpVpnName);
 498                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
 499                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
 500                 confTx.put(getRoutersIdentifier(bgpVpnId), rtrs, CREATE_MISSING_PARENTS);
 501             }
 502             if (create) {
 503                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 504             } else {
 505                 removeDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, confTx);
 506             }
 507             return;
 508         }
 509
 510         //Router ID is used as the internal VPN's name, hence the vrf-id in VpnInstance Op DataStore
 511         addOrDelDefaultFibRouteForSNAT(routerName, routerId, create, confTx);
 512     }
 513
 514     private void addOrDelDefaultFibRouteForSNAT(String routerName, long routerId, boolean create,
 515             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 516         List<BigInteger> switches = naptSwitchSelector.getDpnsForVpn(routerName);
 517         if (switches.isEmpty()) {
 518             LOG.info("addOrDelDefaultFibRouteForSNAT : No switches found for router {}", routerName);
 519             return;
 520         }
 521         if (routerId == NatConstants.INVALID_ID) {
 522             LOG.error("addOrDelDefaultFibRouteForSNAT : Could not retrieve router Id for {} to program "
 523                     + "default NAT route in FIB", routerName);
 524             return;
 525         }
 526         for (BigInteger dpnId : switches) {
 527             if (create) {
 528                 LOG.debug("addOrDelDefaultFibRouteForSNAT : installing default NAT route for router {} in dpn {} "
 529                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 530                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 531             } else {
 532                 LOG.debug("addOrDelDefaultFibRouteForSNAT : removing default NAT route for router {} in dpn {} "
 533                         + "for the internal vpn-id {}", routerId, dpnId, routerId);
 534                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 535             }
 536         }
 537     }
 538
 539     private void addDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 540         long bgpVpnId, TypedWriteTransaction<Configuration> confTx) {
 541         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 542         if (dpnIds.isEmpty()) {
 543             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 544                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 545             return;
 546         }
 547         for (BigInteger dpnId : dpnIds) {
 548             if (bgpVpnId != NatConstants.INVALID_ID) {
 549                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 550                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 551                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 552             } else {
 553                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : installing default NAT route for router {} "
 554                     + "in dpn {} for the internal vpn", routerId, dpnId);
 555                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, confTx);
 556             }
 557         }
 558     }
 559
 560     private void removeDefaultFibRouteForSnatWithBgpVpn(String routerName, long routerId,
 561             long bgpVpnId, TypedReadWriteTransaction<Configuration> confTx)
 562             throws ExecutionException, InterruptedException {
 563         List<BigInteger> dpnIds = NatUtil.getDpnsForRouter(dataBroker, routerName);
 564         if (dpnIds.isEmpty()) {
 565             LOG.error("addOrDelDefaultFibRouteForSNATWIthBgpVpn: No dpns are part of router {} to program "
 566                 + "default NAT flows for BGP-VPN {}", routerName, bgpVpnId);
 567             return;
 568         }
 569         for (BigInteger dpnId : dpnIds) {
 570             if (bgpVpnId != NatConstants.INVALID_ID) {
 571                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 572                     + "in dpn {} for the BGP vpnID {}", routerId, dpnId, bgpVpnId);
 573                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, bgpVpnId, routerId, confTx);
 574             } else {
 575                 LOG.debug("addOrDelDefaultFibRouteForSnatWithBgpVpn : removing default NAT route for router {} "
 576                     + "in dpn {} for the internal vpn", routerId, dpnId);
 577                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, confTx);
 578             }
 579         }
 580     }
 581
 582     protected void installOutboundMissEntry(String routerName, long routerId, BigInteger primarySwitchId,
 583         TypedWriteTransaction<Configuration> confTx) {
 584         LOG.debug("installOutboundMissEntry : Router ID from getVpnId {}", routerId);
 585         if (routerId != NatConstants.INVALID_ID) {
 586             LOG.debug("installOutboundMissEntry : Creating miss entry on primary {}, for router {}",
 587                     primarySwitchId, routerId);
 588             createOutboundTblEntry(primarySwitchId, routerId, confTx);
 589         } else {
 590             LOG.error("installOutboundMissEntry : Unable to fetch Router Id  for RouterName {}, failed to "
 591                 + "createAndInstallMissEntry", routerName);
 592         }
 593     }
 594
 595     public String getFlowRefOutbound(BigInteger dpnId, short tableId, long routerID, int protocol) {
 596         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 597                 .FLOWID_SEPARATOR + routerID + NatConstants.FLOWID_SEPARATOR + protocol;
 598     }
 599
 600     private String getFlowRefNaptPreFib(BigInteger dpnId, short tableId, long vpnId) {
 601         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 602                 .FLOWID_SEPARATOR + vpnId;
 603     }
 604
 605     public BigInteger getCookieOutboundFlow(long routerId) {
 606         return NwConstants.COOKIE_OUTBOUND_NAPT_TABLE.add(new BigInteger("0110001", 16)).add(
 607             BigInteger.valueOf(routerId));
 608     }
 609
 610     private ActionLearn getLearnActionForPunt(int protocol, int hardTimeout, BigInteger cookie) {
 611         long l4SrcPortField;
 612         long l4DstPortField;
 613         int l4portFieldLen = NxmOfFieldType.NXM_OF_TCP_SRC.getFlowModHeaderLenInt();
 614
 615         if (protocol == NwConstants.IP_PROT_TCP) {
 616             l4SrcPortField = NxmOfFieldType.NXM_OF_TCP_SRC.getType();
 617             l4DstPortField = NxmOfFieldType.NXM_OF_TCP_DST.getType();
 618         } else {
 619             l4SrcPortField = NxmOfFieldType.NXM_OF_UDP_SRC.getType();
 620             l4DstPortField = NxmOfFieldType.NXM_OF_UDP_DST.getType();
 621         }
 622         List<ActionLearn.FlowMod> flowMods = Arrays.asList(
 623                 new MatchFromValue(NwConstants.ETHTYPE_IPV4, NxmOfFieldType.NXM_OF_ETH_TYPE.getType(),
 624                         NxmOfFieldType.NXM_OF_ETH_TYPE.getFlowModHeaderLenInt()),
 625                 new MatchFromValue(protocol, NxmOfFieldType.NXM_OF_IP_PROTO.getType(),
 626                         NxmOfFieldType.NXM_OF_IP_PROTO.getFlowModHeaderLenInt()),
 627                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_SRC.getType(), NxmOfFieldType.NXM_OF_IP_SRC.getType(),
 628                         NxmOfFieldType.NXM_OF_IP_SRC.getFlowModHeaderLenInt()),
 629                 new MatchFromField(NxmOfFieldType.NXM_OF_IP_DST.getType(), NxmOfFieldType.NXM_OF_IP_DST.getType(),
 630                         NxmOfFieldType.NXM_OF_IP_DST.getFlowModHeaderLenInt()),
 631                 new MatchFromField(l4SrcPortField, l4SrcPortField, l4portFieldLen),
 632                 new MatchFromField(l4DstPortField, l4DstPortField, l4portFieldLen),
 633                 new MatchFromField(NxmOfFieldType.OXM_OF_METADATA.getType(),
 634                         MetaDataUtil.METADATA_VPN_ID_OFFSET,
 635                         NxmOfFieldType.OXM_OF_METADATA.getType(), MetaDataUtil.METADATA_VPN_ID_OFFSET,
 636                         MetaDataUtil.METADATA_VPN_ID_BITLEN));
 637
 638         return new ActionLearn(0, hardTimeout, 7, cookie, 0,
 639                 NwConstants.OUTBOUND_NAPT_TABLE, 0, 0, flowMods);
 640     }
 641
 642     private FlowEntity buildIcmpDropFlow(BigInteger dpnId, long routerId, long vpnId) {
 643         List<MatchInfo> matches = new ArrayList<>();
 644         matches.add(MatchEthernetType.IPV4);
 645         matches.add(MatchIpProtocol.ICMP);
 646         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 647
 648         List<ActionInfo> actionInfos = new ArrayList<>();
 649         actionInfos.add(new ActionDrop());
 650
 651         List<InstructionInfo> instructions = new ArrayList<>();
 652         instructions.add(new InstructionApplyActions(actionInfos));
 653
 654         String flowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
 655                 NwConstants.IP_PROT_ICMP);
 656         FlowEntity flow = MDSALUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 657                 NwConstants.TABLE_MISS_PRIORITY, "icmp drop flow", 0, 0,
 658                 NwConstants.COOKIE_OUTBOUND_NAPT_TABLE, matches, instructions);
 659         return flow;
 660     }
 661
 662     protected FlowEntity buildOutboundFlowEntity(BigInteger dpId, long routerId, int protocol) {
 663         LOG.debug("buildOutboundFlowEntity : called for dpId {} and routerId{}", dpId, routerId);
 664         BigInteger cookie = getCookieOutboundFlow(routerId);
 665         List<MatchInfo> matches = new ArrayList<>();
 666         matches.add(MatchEthernetType.IPV4);
 667         matches.add(new MatchIpProtocol((short)protocol));
 668         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 669
 670         List<InstructionInfo> instructions = new ArrayList<>();
 671         List<ActionInfo> actionsInfos = new ArrayList<>();
 672         actionsInfos.add(new ActionPuntToController());
 673         if (snatPuntTimeout != 0) {
 674             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
 675         }
 676         instructions.add(new InstructionApplyActions(actionsInfos));
 677
 678         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
 679         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE, flowRef,
 680             5, flowRef, 0, 0,
 681             cookie, matches, instructions);
 682         LOG.debug("installOutboundMissEntry : returning flowEntity {}", flowEntity);
 683         return flowEntity;
 684     }
 685
 686     public void createOutboundTblEntry(BigInteger dpnId, long routerId, TypedWriteTransaction<Configuration> confTx) {
 687         LOG.debug("createOutboundTblEntry : called for dpId {} and routerId {}", dpnId, routerId);
 688         FlowEntity tcpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_TCP);
 689         LOG.debug("createOutboundTblEntry : Installing tcp flow {}", tcpflowEntity);
 690         mdsalManager.addFlow(confTx, tcpflowEntity);
 691
 692         FlowEntity udpflowEntity = buildOutboundFlowEntity(dpnId, routerId, NwConstants.IP_PROT_UDP);
 693         LOG.debug("createOutboundTblEntry : Installing udp flow {}", udpflowEntity);
 694         mdsalManager.addFlow(confTx, udpflowEntity);
 695
 696         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, routerId);
 697         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
 698         mdsalManager.addFlow(confTx, icmpDropFlow);
 699     }
 700
 701     @Nullable
 702     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 703         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 704         RpcResult<GetTunnelInterfaceNameOutput> rpcResult;
 705         try {
 706             Future<RpcResult<GetTunnelInterfaceNameOutput>> result =
 707                 itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 708                     .setSourceDpid(srcDpId)
 709                     .setDestinationDpid(dstDpId)
 710                     .setTunnelType(tunType)
 711                     .build());
 712             rpcResult = result.get();
 713             if (!rpcResult.isSuccessful()) {
 714                 tunType = TunnelTypeGre.class;
 715                 result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 716                     .setSourceDpid(srcDpId)
 717                     .setDestinationDpid(dstDpId)
 718                     .setTunnelType(tunType)
 719                     .build());
 720                 rpcResult = result.get();
 721                 if (!rpcResult.isSuccessful()) {
 722                     LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 723                             rpcResult.getErrors());
 724                 } else {
 725                     return rpcResult.getResult().getInterfaceName();
 726                 }
 727                 LOG.warn("getTunnelInterfaceName : RPC Call to getTunnelInterfaceId returned with Errors {}",
 728                         rpcResult.getErrors());
 729             } else {
 730                 return rpcResult.getResult().getInterfaceName();
 731             }
 732         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 733             LOG.error("getTunnelInterfaceName : Exception when getting tunnel interface Id for tunnel "
 734                     + "between {} and {}", srcDpId, dstDpId, e);
 735         }
 736
 737         return null;
 738     }
 739
 740     protected void installSnatMissEntryForPrimrySwch(BigInteger dpnId, String routerName, long routerId,
 741         TypedWriteTransaction<Configuration> confTx) {
 742
 743         LOG.debug("installSnatMissEntry : called for for the primary NAPT switch dpnId {} ", dpnId);
 744         // Install miss entry pointing to group
 745         FlowEntity flowEntity = buildSnatFlowEntityForPrmrySwtch(dpnId, routerName, routerId);
 746         mdsalManager.addFlow(confTx, flowEntity);
 747     }
 748
 749     protected void installSnatMissEntry(BigInteger dpnId, List<BucketInfo> bucketInfo,
 750             String routerName, long routerId) {
 751         LOG.debug("installSnatMissEntry : called for dpnId {} with primaryBucket {} ",
 752             dpnId, bucketInfo.get(0));
 753         // Install the select group
 754         long groupId = createGroupId(getGroupIdKey(routerName));
 755         GroupEntity groupEntity =
 756             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 757         LOG.debug("installSnatMissEntry : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 758         mdsalManager.syncInstallGroup(groupEntity);
 759         // Install miss entry pointing to group
 760         FlowEntity flowEntity = buildSnatFlowEntity(dpnId, routerName, routerId, groupId);
 761         if (flowEntity == null) {
 762             LOG.error("installSnatMissEntry : Flow entity received as NULL. "
 763                     + "Cannot proceed with installation of SNAT Flow in table {} which is pointing to Group "
 764                     + "on Non NAPT DPN {} for router {}", NwConstants.PSNAT_TABLE, dpnId, routerName);
 765             return;
 766         }
 767         mdsalManager.installFlow(flowEntity);
 768     }
 769
 770     long installGroup(BigInteger dpnId, String routerName, List<BucketInfo> bucketInfo) {
 771         long groupId = createGroupId(getGroupIdKey(routerName));
 772         GroupEntity groupEntity =
 773             MDSALUtil.buildGroupEntity(dpnId, groupId, routerName, GroupTypes.GroupAll, bucketInfo);
 774         LOG.debug("installGroup : installing the SNAT to NAPT GroupEntity:{}", groupEntity);
 775         mdsalManager.syncInstallGroup(groupEntity);
 776         return groupId;
 777     }
 778
 779     private FlowEntity buildSnatFlowEntity(BigInteger dpId, String routerName, long routerId, long groupId) {
 780         LOG.debug("buildSnatFlowEntity : called for dpId {}, routerName {} and groupId {}",
 781                 dpId, routerName, groupId);
 782         List<MatchInfo> matches = new ArrayList<>();
 783         matches.add(MatchEthernetType.IPV4);
 784         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 785
 786         List<ActionInfo> actionsInfo = new ArrayList<>();
 787         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager, idManager,
 788                 routerId, routerName);
 789         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
 790         LOG.debug("buildSnatFlowEntity : Setting the tunnel to the list of action infos {}", actionsInfo);
 791         actionsInfo.add(new ActionGroup(groupId));
 792         List<InstructionInfo> instructions = new ArrayList<>();
 793         instructions.add(new InstructionApplyActions(actionsInfo));
 794         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 795         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 796                 NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 797                 NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 798
 799         LOG.debug("buildSnatFlowEntity : Returning SNAT Flow Entity {}", flowEntity);
 800         return flowEntity;
 801     }
 802
 803     private FlowEntity buildSnatFlowEntityForPrmrySwtch(BigInteger dpId, String routerName, long routerId) {
 804
 805         LOG.debug("buildSnatFlowEntityForPrmrySwtch : called for primary NAPT switch dpId {}, routerName {}", dpId,
 806             routerName);
 807         List<MatchInfo> matches = new ArrayList<>();
 808         matches.add(MatchEthernetType.IPV4);
 809         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(routerId), MetaDataUtil.METADATA_MASK_VRFID));
 810
 811         List<InstructionInfo> instructions = new ArrayList<>();
 812         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 813
 814         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
 815         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 816             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
 817             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
 818
 819         LOG.debug("buildSnatFlowEntityForPrmrySwtch : Returning SNAT Flow Entity {}", flowEntity);
 820         return flowEntity;
 821     }
 822
 823     // TODO : Replace this with ITM Rpc once its available with full functionality
 824     protected void installTerminatingServiceTblEntry(BigInteger dpnId, String routerName, long routerId,
 825         TypedWriteTransaction<Configuration> confTx) {
 826
 827         LOG.debug("installTerminatingServiceTblEntry : for switch {}, routerName {}",
 828             dpnId, routerName);
 829         FlowEntity flowEntity = buildTsFlowEntity(dpnId, routerName, routerId);
 830         if (flowEntity == null) {
 831             LOG.error("installTerminatingServiceTblEntry : Flow entity received as NULL. "
 832                     + "Cannot proceed with installation of Terminating Service table {} which is pointing to table {} "
 833                     + "on DPN {} for router {}", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.OUTBOUND_NAPT_TABLE,
 834                     dpnId, routerName);
 835             return;
 836         }
 837         mdsalManager.addFlow(confTx, flowEntity);
 838
 839     }
 840
 841     private FlowEntity buildTsFlowEntity(BigInteger dpId, String routerName, long routerId) {
 842         List<MatchInfo> matches = new ArrayList<>();
 843         matches.add(MatchEthernetType.IPV4);
 844         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil, elanManager,
 845                 idManager, routerId, routerName);
 846         matches.add(new MatchTunnelId(BigInteger.valueOf(tunnelId)));
 847         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
 848         List<InstructionInfo> instructions = new ArrayList<>();
 849         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(routerId),
 850                 MetaDataUtil.METADATA_MASK_VRFID));
 851         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
 852         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
 853                 NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0, NwConstants.COOKIE_TS_TABLE, matches,
 854                 instructions);
 855         return flowEntity;
 856     }
 857
 858     public String getFlowRefTs(BigInteger dpnId, short tableId, long routerID) {
 859         return NatConstants.NAPT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 860                 .FLOWID_SEPARATOR + routerID;
 861     }
 862
 863     public static String getFlowRefSnat(BigInteger dpnId, short tableId, String routerID) {
 864         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NatConstants.FLOWID_SEPARATOR + tableId + NatConstants
 865                 .FLOWID_SEPARATOR + routerID;
 866     }
 867
 868     private String getGroupIdKey(String routerName) {
 869         return "snatmiss." + routerName;
 870     }
 871
 872     protected long createGroupId(String groupIdKey) {
 873         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 874             .setPoolName(NatConstants.SNAT_IDPOOL_NAME).setIdKey(groupIdKey)
 875             .build();
 876         try {
 877             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 878             RpcResult<AllocateIdOutput> rpcResult = result.get();
 879             return rpcResult.getResult().getIdValue();
 880         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 881             LOG.error("Exception While allocating id for group: {}", groupIdKey, e);
 882         }
 883         return 0;
 884     }
 885
 886     protected void handleSwitches(BigInteger dpnId, String routerName, long routerId, BigInteger primarySwitchId) {
 887         LOG.debug("handleSwitches : Installing SNAT miss entry in switch {}", dpnId);
 888         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 889         String ifNamePrimary = getTunnelInterfaceName(dpnId, primarySwitchId);
 890         List<BucketInfo> listBucketInfo = new ArrayList<>();
 891
 892         if (ifNamePrimary != null) {
 893             LOG.debug("handleSwitches : On Non- Napt switch , Primary Tunnel interface is {}", ifNamePrimary);
 894             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 895                     interfaceManager, ifNamePrimary, routerId, true);
 896             if (listActionInfoPrimary.isEmpty()) {
 897                 LOG.error("handleSwitches : Unable to retrieve output actions on Non-NAPT switch {} for router {}"
 898                         + " towards Napt-switch {} via tunnel interface {}", dpnId, routerName, primarySwitchId,
 899                         ifNamePrimary);
 900             }
 901         } else {
 902             LOG.error("handleSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} from "
 903                     + "Non-Napt switch {} for router {}", primarySwitchId, dpnId, routerName);
 904         }
 905         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 906
 907         listBucketInfo.add(0, bucketPrimary);
 908         installSnatMissEntry(dpnId, listBucketInfo, routerName, routerId);
 909     }
 910
 911     List<BucketInfo> getBucketInfoForNonNaptSwitches(BigInteger nonNaptSwitchId,
 912                                                      BigInteger primarySwitchId, String routerName, long routerId) {
 913         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 914         String ifNamePrimary = getTunnelInterfaceName(nonNaptSwitchId, primarySwitchId);
 915         List<BucketInfo> listBucketInfo = new ArrayList<>();
 916
 917         if (ifNamePrimary != null) {
 918             LOG.debug("getBucketInfoForNonNaptSwitches : On Non- Napt switch , Primary Tunnel interface is {}",
 919                     ifNamePrimary);
 920             listActionInfoPrimary = NatUtil.getEgressActionsForInterface(odlInterfaceRpcService, itmManager,
 921                     interfaceManager, ifNamePrimary, routerId, true);
 922             if (listActionInfoPrimary.isEmpty()) {
 923                 LOG.error("getBucketInfoForNonNaptSwitches : Unable to retrieve output actions on Non-NAPT switch {} "
 924                         + "for router {} towards Napt-switch {} via tunnel interface {}",
 925                         nonNaptSwitchId, routerName, primarySwitchId, ifNamePrimary);
 926             }
 927         } else {
 928             LOG.error("getBucketInfoForNonNaptSwitches : Unable to obtain primary tunnel interface to Napt-Switch {} "
 929                     + "from Non-Napt switch {} for router {}", primarySwitchId, nonNaptSwitchId, routerName);
 930         }
 931         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 932
 933         listBucketInfo.add(0, bucketPrimary);
 934         return listBucketInfo;
 935     }
 936
 937     protected void handlePrimaryNaptSwitch(BigInteger dpnId, String routerName, long routerId,
 938         TypedWriteTransaction<Configuration> confTx) {
 939
 940        /*
 941         * Primary NAPT Switch â€“ bucket Should always point back to its own Outbound Table
 942         */
 943
 944         LOG.debug("handlePrimaryNaptSwitch : Installing SNAT miss entry in Primary NAPT switch {} ", dpnId);
 945
 946 /*
 947         List<BucketInfo> listBucketInfo = new ArrayList<>();
 948         List<ActionInfo> listActionInfoPrimary = new ArrayList<>();
 949         listActionInfoPrimary.add(new ActionNxResubmit(NatConstants.TERMINATING_SERVICE_TABLE));
 950         BucketInfo bucketPrimary = new BucketInfo(listActionInfoPrimary);
 951         listBucketInfo.add(0, bucketPrimary);
 952 */
 953
 954         installSnatMissEntryForPrimrySwch(dpnId, routerName, routerId, confTx);
 955         installTerminatingServiceTblEntry(dpnId, routerName, routerId, confTx);
 956         //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the router ID.
 957         installNaptPfibEntry(dpnId, routerId, confTx);
 958         Uuid networkId = NatUtil.getNetworkIdFromRouterId(dataBroker, routerId);
 959         if (networkId != null) {
 960             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
 961             if (vpnUuid != null) {
 962                 long vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 963                 coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 964                     installNaptPfibEntriesForExternalSubnets(routerName, dpnId, null);
 965                     //Install the NAPT PFIB TABLE which forwards outgoing packet to FIB Table matching on the VPN ID.
 966                     if (vpnId != NatConstants.INVALID_ID) {
 967                         installNaptPfibEntry(dpnId, vpnId, null);
 968                     }
 969                     return Collections.emptyList();
 970                 });
 971             } else {
 972                 LOG.warn("handlePrimaryNaptSwitch : External Vpn ID missing for Ext-Network : {}", networkId);
 973             }
 974         } else {
 975             LOG.warn("handlePrimaryNaptSwitch : External Network not available for router : {}", routerName);
 976         }
 977     }
 978
 979     public void installNaptPfibEntry(BigInteger dpnId, long segmentId,
 980             @Nullable TypedWriteTransaction<Configuration> confTx) {
 981         LOG.debug("installNaptPfibEntry : called for dpnId {} and segmentId {} ", dpnId, segmentId);
 982         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntity(dpnId, segmentId);
 983         if (confTx != null) {
 984             mdsalManager.addFlow(confTx, naptPfibFlowEntity);
 985         } else {
 986             mdsalManager.installFlow(naptPfibFlowEntity);
 987         }
 988     }
 989
 990     public FlowEntity buildNaptPfibFlowEntity(BigInteger dpId, long segmentId) {
 991
 992         LOG.debug("buildNaptPfibFlowEntity : called for dpId {}, segmentId {}", dpId, segmentId);
 993         List<MatchInfo> matches = new ArrayList<>();
 994         matches.add(MatchEthernetType.IPV4);
 995         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 996
 997         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
 998         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
 999         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
1000         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
1001         instructionInfo.add(new InstructionApplyActions(listActionInfo));
1002
1003         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
1004         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
1005             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
1006             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
1007         LOG.debug("buildNaptPfibFlowEntity : Returning NaptPFib Flow Entity {}", flowEntity);
1008         return flowEntity;
1009     }
1010
1011     public void handleSnatReverseTraffic(TypedWriteTransaction<Configuration> confTx, BigInteger dpnId, Routers router,
1012         long routerId, String routerName, String externalIp) {
1013         LOG.debug("handleSnatReverseTraffic : entry for DPN ID {}, routerId {}, externalIp: {}",
1014             dpnId, routerId, externalIp);
1015         Uuid networkId = router.getNetworkId();
1016         if (networkId == null) {
1017             LOG.error("handleSnatReverseTraffic : networkId is null for the router ID {}", routerId);
1018             return;
1019         }
1020         final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1021         if (vpnName == null) {
1022             LOG.error("handleSnatReverseTraffic : No VPN associated with ext nw {} to handle add external ip "
1023                     + "configuration {} in router {}", networkId, externalIp, routerId);
1024             return;
1025         }
1026         advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1027             externalIp, networkId, router, confTx);
1028         LOG.debug("handleSnatReverseTraffic : exit for DPN ID {}, routerId {}, externalIp : {}",
1029             dpnId, routerId, externalIp);
1030     }
1031
1032     public void advToBgpAndInstallFibAndTsFlows(final BigInteger dpnId, final short tableId, final String vpnName,
1033                                                 final long routerId, final String routerName, final String externalIp,
1034                                                 final Uuid extNetworkId, @Nullable final Routers router,
1035                                                 final TypedWriteTransaction<Configuration> confTx) {
1036         LOG.debug("advToBgpAndInstallFibAndTsFlows : entry for DPN ID {}, tableId {}, vpnname {} "
1037                 + "and externalIp {}", dpnId, tableId, vpnName, externalIp);
1038         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
1039         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
1040         if (rd == null || rd.isEmpty()) {
1041             LOG.error("advToBgpAndInstallFibAndTsFlows : Unable to get RD for VPN Name {}", vpnName);
1042             return;
1043         }
1044         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
1045         if (extNwProvType == null) {
1046             LOG.error("advToBgpAndInstallFibAndTsFlows : External Network Provider Type missing");
1047             return;
1048         }
1049         if (extNwProvType == ProviderTypes.VXLAN) {
1050             evpnSnatFlowProgrammer.evpnAdvToBgpAndInstallFibAndTsFlows(dpnId, tableId, externalIp, vpnName, rd,
1051                 nextHopIp, routerId, routerName, extNetworkId, confTx);
1052             return;
1053         }
1054         //Generate VPN label for the external IP
1055         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
1056             .setIpPrefix(externalIp).build();
1057         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
1058
1059         //On successful generation of the VPN label, advertise the route to the BGP and install the FIB routes.
1060         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
1061             if (result.isSuccessful()) {
1062                 LOG.debug("advToBgpAndInstallFibAndTsFlows : inside apply with result success");
1063                 GenerateVpnLabelOutput output = result.getResult();
1064                 final long label = output.getLabel();
1065
1066                 int externalIpInDsFlag = 0;
1067                 //Get IPMaps from the DB for the router ID
1068                 List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
1069                 for (IpMap dbIpMap : dbIpMaps) {
1070                     String dbExternalIp = dbIpMap.getExternalIp();
1071                     //Select the IPMap, whose external IP is the IP for which FIB is installed
1072                     if (dbExternalIp.contains(externalIp)) {
1073                         String dbInternalIp = dbIpMap.getInternalIp();
1074                         IpMapKey dbIpMapKey = dbIpMap.key();
1075                         LOG.debug("advToBgpAndInstallFibAndTsFlows : Setting label {} for internalIp {} "
1076                                 + "and externalIp {}", label, dbInternalIp, externalIp);
1077                         IpMap newIpm = new IpMapBuilder().withKey(dbIpMapKey).setInternalIp(dbInternalIp)
1078                             .setExternalIp(dbExternalIp).setLabel(label).build();
1079                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1080                             naptManager.getIpMapIdentifier(routerId, dbInternalIp), newIpm);
1081                         externalIpInDsFlag++;
1082                     }
1083                 }
1084                 if (externalIpInDsFlag <= 0) {
1085                     LOG.debug("advToBgpAndInstallFibAndTsFlows : External Ip {} not found in DS, "
1086                             + "Failed to update label {} for routerId {} in DS",
1087                             externalIp, label, routerId);
1088                     String errMsg = String.format("Failed to update label %s due to external Ip %s not"
1089                         + " found in DS for router %s", label, externalIp, routerId);
1090                     return Futures.immediateFailedFuture(new Exception(errMsg));
1091                 }
1092                 //Inform BGP
1093                 long l3vni = 0;
1094                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1095                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
1096                 }
1097                 Routers extRouter = router != null ? router :
1098                     NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1099                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
1100                     externalIp, nextHopIp, extRouter.getNetworkId().getValue(), null, label, l3vni,
1101                     RouteOrigin.STATIC, dpnId);
1102
1103                 //Install custom FIB routes
1104                 List<Instruction> tunnelTableCustomInstructions = new ArrayList<>();
1105                 tunnelTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(0));
1106                 makeTunnelTableEntry(dpnId, label, l3vni, tunnelTableCustomInstructions, confTx,
1107                         extNwProvType);
1108                 makeLFibTableEntry(dpnId, label, tableId, confTx);
1109
1110                 //Install custom FIB routes - FIB table.
1111                 List<Instruction> fibTableCustomInstructions = createFibTableCustomInstructions(tableId,
1112                         routerName, externalIp);
1113                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1114                     //Install the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
1115                     NatUtil.makePreDnatToSnatTableEntry(mdsalManager, dpnId, NwConstants.INBOUND_NAPT_TABLE, confTx);
1116                 }
1117                 String externalVpn = vpnName;
1118                 Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp, extRouter);
1119                 if (extNwProvType == ProviderTypes.VLAN || extNwProvType == ProviderTypes.FLAT) {
1120                     Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
1121                             externalSubnetId);
1122                     if (externalSubnet.isPresent()) {
1123                         externalVpn =  externalSubnetId.getValue();
1124                     }
1125                 }
1126                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
1127                 CreateFibEntryInput input = new CreateFibEntryInputBuilder()
1128                     .setVpnName(externalVpn)
1129                     .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
1130                     .setIpAddressSource(CreateFibEntryInput.IpAddressSource.ExternalFixedIP)
1131                     .setInstruction(fibTableCustomInstructions).build();
1132                 return fibService.createFibEntry(input);
1133             } else {
1134                 LOG.error("advToBgpAndInstallFibAndTsFlows : inside apply with result failed");
1135                 String errMsg = String.format("Could not retrieve the label for prefix %s in VPN %s, %s",
1136                     externalIp, vpnName, result.getErrors());
1137                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
1138             }
1139         }, MoreExecutors.directExecutor());
1140
1141         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
1142
1143             @Override
1144             public void onFailure(@NonNull Throwable error) {
1145                 LOG.error("advToBgpAndInstallFibAndTsFlows : Error in generate label or fib install process", error);
1146             }
1147
1148             @Override
1149             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
1150                 if (result.isSuccessful()) {
1151                     LOG.info("advToBgpAndInstallFibAndTsFlows : Successfully installed custom FIB routes for prefix {}",
1152                             externalIp);
1153                 } else {
1154                     LOG.error("advToBgpAndInstallFibAndTsFlows : Error in rpc call to create custom Fib entries "
1155                             + "for prefix {} in DPN {}, {}", externalIp, dpnId, result.getErrors());
1156                 }
1157             }
1158         }, MoreExecutors.directExecutor());
1159     }
1160
1161     private List<Instruction> createFibTableCustomInstructions(short tableId, String routerName,
1162             String externalIp) {
1163         List<Instruction> fibTableCustomInstructions = new ArrayList<>();
1164         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
1165         long externalSubnetVpnId = NatUtil.getExternalSubnetVpnIdForRouterExternalIp(dataBroker,
1166                 externalIp, router);
1167         int instructionIndex = 0;
1168         if (externalSubnetVpnId != NatConstants.INVALID_ID) {
1169             BigInteger subnetIdMetaData = MetaDataUtil.getVpnIdMetadata(externalSubnetVpnId);
1170             fibTableCustomInstructions.add(new InstructionWriteMetadata(subnetIdMetaData,
1171                     MetaDataUtil.METADATA_MASK_VRFID).buildInstruction(instructionIndex));
1172             instructionIndex++;
1173         }
1174
1175         fibTableCustomInstructions.add(new InstructionGotoTable(tableId).buildInstruction(instructionIndex));
1176         return fibTableCustomInstructions;
1177     }
1178
1179     private void makeLFibTableEntry(BigInteger dpId, long serviceId, short tableId,
1180         TypedWriteTransaction<Configuration> confTx) {
1181         List<MatchInfo> matches = new ArrayList<>();
1182         matches.add(MatchEthernetType.MPLS_UNICAST);
1183         matches.add(new MatchMplsLabel(serviceId));
1184
1185         List<Instruction> instructions = new ArrayList<>();
1186         List<ActionInfo> actionsInfos = new ArrayList<>();
1187         //NAT is required for IPv4 only. Hence always etherType will be IPv4
1188         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
1189         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
1190         instructions.add(writeInstruction);
1191         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
1192
1193         // Install the flow entry in L3_LFIB_TABLE
1194         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
1195
1196         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
1197             10, flowRef, 0, 0,
1198             COOKIE_VM_LFIB_TABLE, matches, instructions);
1199
1200         mdsalManager.addFlow(confTx, dpId, flowEntity);
1201
1202         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
1203     }
1204
1205     private void makeTunnelTableEntry(BigInteger dpnId, long serviceId, long l3Vni,
1206         List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx,
1207         ProviderTypes extNwProvType) {
1208
1209         List<MatchInfo> mkMatches = new ArrayList<>();
1210
1211         LOG.debug("makeTunnelTableEntry : DpnId = {} and serviceId = {} and actions = {}",
1212             dpnId, serviceId, customInstructions);
1213
1214         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1215             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
1216         } else {
1217             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
1218         }
1219
1220         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
1221             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), 5,
1222             String.format("%s:%d", "TST Flow Entry ", serviceId),
1223             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
1224
1225         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
1226     }
1227
1228     protected InstanceIdentifier<RouterIds> getRoutersIdentifier(long routerId) {
1229         return InstanceIdentifier.builder(RouterIdName.class).child(RouterIds.class,
1230             new RouterIdsKey(routerId)).build();
1231     }
1232
1233     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
1234         return NatConstants.SNAT_FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants
1235                 .FLOWID_SEPARATOR + id + NwConstants.FLOWID_SEPARATOR + ipAddress;
1236     }
1237
1238     @Override
1239     protected void update(InstanceIdentifier<Routers> identifier, Routers original, Routers update) {
1240         String routerName = original.getRouterName();
1241         Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1242         if (routerId == NatConstants.INVALID_ID) {
1243             LOG.error("update : external router event - Invalid routerId for routerName {}", routerName);
1244             return;
1245         }
1246         // Check if its update on SNAT flag
1247         boolean originalSNATEnabled = original.isEnableSnat();
1248         boolean updatedSNATEnabled = update.isEnableSnat();
1249         LOG.debug("update :called with originalFlag and updatedFlag for SNAT enabled "
1250             + "as {} and {}", originalSNATEnabled, updatedSNATEnabled);
1251         /* Get Primary Napt Switch for existing router from "router-to-napt-switch" DS.
1252          * if dpnId value is null or zero then go for electing new Napt switch for existing router.
1253          */
1254         long bgpVpnId = NatConstants.INVALID_ID;
1255         Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1256         if (bgpVpnUuid != null) {
1257             bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1258         }
1259         BigInteger dpnId = getPrimaryNaptSwitch(routerName);
1260         final long finalBgpVpnId = bgpVpnId;
1261         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + update.key(), () -> {
1262             List<ListenableFuture<Void>> futures = new ArrayList<>();
1263             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeFlowInvTx -> {
1264                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, removeFlowInvTx -> {
1265                     Uuid networkId = original.getNetworkId();
1266                     if (originalSNATEnabled != updatedSNATEnabled) {
1267                         if (originalSNATEnabled) {
1268                             if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1269                                 // Router has no interface attached
1270                                 return;
1271                             }
1272                             //SNAT disabled for the router
1273                             Uuid networkUuid = original.getNetworkId();
1274                             LOG.info("update : SNAT disabled for Router {}", routerName);
1275                             Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1276                             handleDisableSnat(original, networkUuid, externalIps, false, null, dpnId, routerId,
1277                                     removeFlowInvTx);
1278                         } else {
1279                             LOG.info("update : SNAT enabled for Router {}", original.getRouterName());
1280                             addOrDelDefFibRouteToSNAT(routerName, routerId, finalBgpVpnId, bgpVpnUuid,
1281                                     true, writeFlowInvTx);
1282                             handleEnableSnat(update, routerId, dpnId, finalBgpVpnId, writeFlowInvTx);
1283                         }
1284                     }
1285                     if (!Objects.equals(original.getExtGwMacAddress(), update.getExtGwMacAddress())) {
1286                         NatUtil.installRouterGwFlows(txRunner, vpnManager, original, dpnId, NwConstants.DEL_FLOW);
1287                         NatUtil.installRouterGwFlows(txRunner, vpnManager, update, dpnId, NwConstants.ADD_FLOW);
1288                     }
1289
1290                     if (updatedSNATEnabled != originalSNATEnabled) {
1291                         LOG.info("update : no need to process external/subnet changes as it's will taken care in "
1292                                 + "handleDisableSnat/handleEnableSnat");
1293                         return;
1294                     }
1295                     //Check if the Update is on External IPs
1296                     LOG.debug("update : Checking if this is update on External IPs");
1297                     List<String> originalExternalIps = NatUtil.getIpsListFromExternalIps(original.getExternalIps());
1298                     List<String> updatedExternalIps = NatUtil.getIpsListFromExternalIps(update.getExternalIps());
1299
1300                     //Check if the External IPs are removed during the update.
1301                     Set<String> removedExternalIps = new HashSet<>(originalExternalIps);
1302                     removedExternalIps.removeAll(updatedExternalIps);
1303                     if (removedExternalIps.size() > 0) {
1304                         LOG.debug("update : Start processing of the External IPs removal during the update "
1305                                 + "operation");
1306                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
1307                                 removedExternalIps, original.getExtGwMacAddress(),
1308                                 dpnId, networkId);
1309
1310                         for (String removedExternalIp : removedExternalIps) {
1311                 /*
1312                     1) Remove the mappings in the IntExt IP model which has external IP.
1313                     2) Remove the external IP in the ExternalCounter model.
1314                     3) For the corresponding subnet IDs whose external IP mapping was removed, allocate one of the
1315                        least loaded external IP.
1316                        Store the subnet IP and the reallocated external IP mapping in the IntExtIp model.
1317                     4) Increase the count of the allocated external IP by one.
1318                     5) Advertise to the BGP if external IP is allocated for the first time for the router
1319                      i.e. the route for the external IP is absent.
1320                     6) Remove the NAPT translation entries from Inbound and Outbound NAPT tables for
1321                      the removed external IPs and also from the model.
1322                     7) Advertise to the BGP for removing the route for the removed external IPs.
1323                  */
1324
1325                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(removedExternalIp);
1326                             String externalIp = externalIpParts[0];
1327                             String externalIpPrefix = externalIpParts[1];
1328                             String externalIpAddrStr = externalIp + "/" + externalIpPrefix;
1329
1330                             LOG.debug("update : Clear the routes from the BGP and remove the FIB and TS "
1331                                     + "entries for removed external IP {}", externalIpAddrStr);
1332                             Uuid vpnUuId = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1333                             String vpnName = "";
1334                             if (vpnUuId != null) {
1335                                 vpnName = vpnUuId.getValue();
1336                             }
1337                             clrRtsFromBgpAndDelFibTs(dpnId, routerId, externalIpAddrStr, vpnName, networkId,
1338                                     update.getExtGwMacAddress(), removeFlowInvTx);
1339
1340                             LOG.debug("update : Remove the mappings in the IntExtIP model which has external IP.");
1341                             //Get the internal IPs which are associated to the removed external IPs
1342                             List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1343                             List<String> removedInternalIps = new ArrayList<>();
1344                             for (IpMap ipMap : ipMaps) {
1345                                 if (ipMap.getExternalIp().equals(externalIpAddrStr)) {
1346                                     removedInternalIps.add(ipMap.getInternalIp());
1347                                 }
1348                             }
1349
1350                             LOG.debug("update : Remove the mappings of the internal IPs from the IntExtIP model.");
1351                             for (String removedInternalIp : removedInternalIps) {
1352                                 LOG.debug("update : Remove the IP mapping of the internal IP {} for the "
1353                                                 + "router ID {} from the IntExtIP model",
1354                                         removedInternalIp, routerId);
1355                                 naptManager.removeFromIpMapDS(routerId, removedInternalIp);
1356                             }
1357
1358                             LOG.debug("update : Remove the count mapping of the external IP {} for the "
1359                                             + "router ID {} from the ExternalIpsCounter model.",
1360                                     externalIpAddrStr, routerId);
1361                             naptManager.removeExternalIpCounter(routerId, externalIpAddrStr);
1362
1363                             LOG.debug("update : Allocate the least loaded external IPs to the subnets "
1364                                     + "whose external IPs were removed.");
1365                             for (String removedInternalIp : removedInternalIps) {
1366                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId,
1367                                         removedInternalIp, writeFlowInvTx);
1368                             }
1369                             LOG.debug("update : Remove the NAPT translation entries from "
1370                                     + "Inbound and Outbound NAPT tables for the removed external IPs.");
1371                             //Get the internalIP and internal Port which were associated to the removed external IP.
1372                             Map<ProtocolTypes, List<String>> protoTypesIntIpPortsMap = new HashMap<>();
1373                             InstanceIdentifier<IpPortMapping> ipPortMappingId = InstanceIdentifier
1374                                     .builder(IntextIpPortMap.class)
1375                                     .child(IpPortMapping.class, new IpPortMappingKey(routerId)).build();
1376                             Optional<IpPortMapping> ipPortMapping;
1377                             try {
1378                                 ipPortMapping = SingleTransactionDataBroker
1379                                             .syncReadOptional(dataBroker,
1380                                                     LogicalDatastoreType.CONFIGURATION, ipPortMappingId);
1381                             } catch (ReadFailedException e) {
1382                                 LOG.error("Failed to read ipPortMapping for router id {}", routerId, e);
1383                                 ipPortMapping = Optional.absent();
1384                             }
1385
1386                             if (ipPortMapping.isPresent()) {
1387                                 for (IntextIpProtocolType intextIpProtocolType :
1388                                         ipPortMapping.get().nonnullIntextIpProtocolType()) {
1389                                     ProtocolTypes protoType = intextIpProtocolType.getProtocol();
1390                                     for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
1391                                         IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
1392                                         if (ipPortExternal.getIpAddress().equals(externalIp)) {
1393                                             List<String> removedInternalIpPorts =
1394                                                     protoTypesIntIpPortsMap.get(protoType);
1395                                             if (removedInternalIpPorts != null) {
1396                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1397                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1398                                             } else {
1399                                                 removedInternalIpPorts = new ArrayList<>();
1400                                                 removedInternalIpPorts.add(ipPortMap.getIpPortInternal());
1401                                                 protoTypesIntIpPortsMap.put(protoType, removedInternalIpPorts);
1402                                             }
1403                                         }
1404                                     }
1405                                 }
1406                             }
1407
1408                             //Remove the IP port map from the intext-ip-port-map model, which were containing
1409                             // the removed external IP.
1410                             Set<Map.Entry<ProtocolTypes, List<String>>> protoTypesIntIpPorts =
1411                                     protoTypesIntIpPortsMap.entrySet();
1412                             Map<String, List<String>> internalIpPortMap = new HashMap<>();
1413                             for (Map.Entry protoTypesIntIpPort : protoTypesIntIpPorts) {
1414                                 ProtocolTypes protocolType = (ProtocolTypes) protoTypesIntIpPort.getKey();
1415                                 List<String> removedInternalIpPorts = (List<String>) protoTypesIntIpPort.getValue();
1416                                 for (String removedInternalIpPort : removedInternalIpPorts) {
1417                                     // Remove the IP port map from the intext-ip-port-map model,
1418                                     // which were containing the removed external IP
1419                                     naptManager.removeFromIpPortMapDS(routerId, removedInternalIpPort,
1420                                             protocolType);
1421                                     //Remove the IP port incomint packer map.
1422                                     naptPacketInHandler.removeIncomingPacketMap(
1423                                             routerId + NatConstants.COLON_SEPARATOR + removedInternalIpPort);
1424                                     String[] removedInternalIpPortParts = removedInternalIpPort
1425                                             .split(NatConstants.COLON_SEPARATOR);
1426                                     if (removedInternalIpPortParts.length == 2) {
1427                                         String removedInternalIp = removedInternalIpPortParts[0];
1428                                         String removedInternalPort = removedInternalIpPortParts[1];
1429                                         List<String> removedInternalPortsList =
1430                                                 internalIpPortMap.get(removedInternalPort);
1431                                         if (removedInternalPortsList != null) {
1432                                             removedInternalPortsList.add(removedInternalPort);
1433                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1434                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1435                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1436                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1437                                             //Remove the NAPT translation entries from Outbound NAPT table
1438                                             naptEventHandler.removeNatFlows(dpnId,
1439                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1440                                                 routerId, removedInternalIp,
1441                                                 Integer.parseInt(removedInternalPort),
1442                                                 protocolType.getName());
1443                                             naptEventHandler.removeNatFlows(dpnId,
1444                                                 NwConstants.INBOUND_NAPT_TABLE,
1445                                                 routerId, removedInternalIp,
1446                                                 Integer.parseInt(removedInternalPort),
1447                                                 protocolType.getName());
1448                                         } else {
1449                                             removedInternalPortsList = new ArrayList<>();
1450                                             removedInternalPortsList.add(removedInternalPort);
1451                                             internalIpPortMap.put(removedInternalIp, removedInternalPortsList);
1452                                             naptPacketInHandler.removeIncomingPacketMap(routerId
1453                                                 + NatConstants.COLON_SEPARATOR + removedInternalIp
1454                                                 + NatConstants.COLON_SEPARATOR + removedInternalPort);
1455                                             //Remove the NAPT translation entries from Outbound NAPT table
1456                                             naptEventHandler.removeNatFlows(dpnId,
1457                                                 NwConstants.OUTBOUND_NAPT_TABLE,
1458                                                 routerId, removedInternalIp,
1459                                                 Integer.parseInt(removedInternalPort),
1460                                                 protocolType.getName());
1461                                             naptEventHandler.removeNatFlows(dpnId,
1462                                                 NwConstants.INBOUND_NAPT_TABLE, routerId, removedInternalIp,
1463                                                 Integer.parseInt(removedInternalPort),
1464                                                 protocolType.getName());
1465                                         }
1466                                     }
1467                                 }
1468                             }
1469
1470                             // Delete the entry from SnatIntIpPortMap DS
1471                             Set<String> internalIps = internalIpPortMap.keySet();
1472                             for (String internalIp : internalIps) {
1473                                 LOG.debug("update : Removing IpPort having the internal IP {} from the "
1474                                         + "model SnatIntIpPortMap", internalIp);
1475                                 naptManager.removeFromSnatIpPortDS(routerId, internalIp);
1476                             }
1477
1478                             naptManager.removeNaptPortPool(externalIp);
1479                         }
1480                         LOG.debug(
1481                                 "update : End processing of the External IPs removal during the update operation");
1482                     }
1483
1484                     //Check if the External IPs are added during the update.
1485                     Set<String> addedExternalIps = new HashSet<>(updatedExternalIps);
1486                     addedExternalIps.removeAll(originalExternalIps);
1487                     if (addedExternalIps.size() != 0) {
1488                         LOG.debug("update : Start processing of the External IPs addition during the update "
1489                                 + "operation");
1490                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName, addedExternalIps,
1491                                 update.getExtGwMacAddress(), dpnId,
1492                                 update.getNetworkId());
1493
1494                         for (String addedExternalIp : addedExternalIps) {
1495                 /*
1496                     1) Do nothing in the IntExtIp model.
1497                     2) Initialise the count of the added external IP to 0 in the ExternalCounter model.
1498                  */
1499                             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(addedExternalIp);
1500                             String externalIp = externalIpParts[0];
1501                             String externalIpPrefix = externalIpParts[1];
1502                             String externalpStr = externalIp + "/" + externalIpPrefix;
1503                             LOG.debug("update : Initialise the count mapping of the external IP {} for the "
1504                                             + "router ID {} in the ExternalIpsCounter model.",
1505                                     externalpStr, routerId);
1506                             naptManager.initialiseNewExternalIpCounter(routerId, externalpStr);
1507                             subnetRegisterMapping(update, routerId);
1508                             LOG.info("update : Installing fib flow fo newly added Ips");
1509                             handleSnatReverseTraffic(writeFlowInvTx, dpnId, update, routerId, routerName, externalpStr);
1510                         }
1511                         LOG.debug(
1512                                 "update : End processing of the External IPs addition during the update operation");
1513                     }
1514
1515                     //Check if its Update on subnets
1516                     LOG.debug("update : Checking if this is update on subnets");
1517                     List<Uuid> originalSubnetIds = original.getSubnetIds();
1518                     List<Uuid> updatedSubnetIds = update.getSubnetIds();
1519                     Set<Uuid> addedSubnetIds =
1520                         updatedSubnetIds != null ? new HashSet<>(updatedSubnetIds) : new HashSet<>();
1521                     if (originalSubnetIds != null) {
1522                         addedSubnetIds.removeAll(originalSubnetIds);
1523                     }
1524
1525                     //Check if the Subnet IDs are added during the update.
1526                     if (addedSubnetIds.size() != 0) {
1527                         LOG.debug(
1528                                 "update : Start processing of the Subnet IDs addition during the update operation");
1529                         for (Uuid addedSubnetId : addedSubnetIds) {
1530                 /*
1531                     1) Select the least loaded external IP for the subnet and store the mapping of the
1532                     subnet IP and the external IP in the IntExtIp model.
1533                     2) Increase the count of the selected external IP by one.
1534                     3) Advertise to the BGP if external IP is allocated for the first time for the
1535                     router i.e. the route for the external IP is absent.
1536                  */
1537                             String subnetIp = NatUtil.getSubnetIp(dataBroker, addedSubnetId);
1538                             if (subnetIp != null) {
1539                                 allocateExternalIp(dpnId, update, routerId, routerName, networkId, subnetIp,
1540                                         writeFlowInvTx);
1541                             }
1542                         }
1543                         LOG.debug("update : End processing of the Subnet IDs addition during the update operation");
1544                     }
1545
1546                     //Check if the Subnet IDs are removed during the update.
1547                     Set<Uuid> removedSubnetIds = new HashSet<>(originalSubnetIds);
1548                     removedSubnetIds.removeAll(updatedSubnetIds);
1549                     if (removedSubnetIds.size() != 0) {
1550                         LOG.debug(
1551                                 "update : Start processing of the Subnet IDs removal during the update operation");
1552                         for (Uuid removedSubnetId : removedSubnetIds) {
1553                             String[] subnetAddr = NatUtil.getSubnetIpAndPrefix(dataBroker, removedSubnetId);
1554                             if (subnetAddr != null) {
1555                     /*
1556                         1) Remove the subnet IP and the external IP in the IntExtIp map
1557                         2) Decrease the count of the coresponding external IP by one.
1558                         3) Advertise to the BGP for removing the routes of the corresponding external
1559                         IP if its not allocated to any other internal IP.
1560                     */
1561
1562                                 String externalIp = naptManager.getExternalIpAllocatedForSubnet(routerId,
1563                                         subnetAddr[0] + "/" + subnetAddr[1]);
1564                                 if (externalIp == null) {
1565                                     LOG.error("update : No mapping found for router ID {} and internal IP {}",
1566                                             routerId, subnetAddr[0]);
1567                                     return;
1568                                 }
1569
1570                                 naptManager.updateCounter(routerId, externalIp, false);
1571                                 // Traverse entire model of external-ip counter whether external ip is not
1572                                 // used by any other internal ip in any router
1573                                 if (!isExternalIpAllocated(externalIp)) {
1574                                     LOG.debug("update : external ip is not allocated to any other "
1575                                             + "internal IP so proceeding to remove routes");
1576                                     clrRtsFromBgpAndDelFibTs(dpnId, routerId, networkId,
1577                                             Collections.singleton(externalIp), null, update.getExtGwMacAddress(),
1578                                             removeFlowInvTx);
1579                                     LOG.debug("update : Successfully removed fib entries in switch {} for "
1580                                                     + "router {} with networkId {} and externalIp {}",
1581                                             dpnId, routerId, networkId, externalIp);
1582                                 }
1583
1584                                 LOG.debug("update : Remove the IP mapping for the router ID {} and "
1585                                         + "internal IP {} external IP {}", routerId, subnetAddr[0], externalIp);
1586                                 naptManager.removeIntExtIpMapDS(routerId, subnetAddr[0] + "/" + subnetAddr[1]);
1587                             }
1588                         }
1589                         LOG.debug("update : End processing of the Subnet IDs removal during the update operation");
1590                     }
1591                 }));
1592             }));
1593             return futures;
1594         }, NatConstants.NAT_DJC_MAX_RETRIES);
1595     }
1596
1597     private boolean isExternalIpAllocated(String externalIp) {
1598         InstanceIdentifier<ExternalIpsCounter> id = InstanceIdentifier.builder(ExternalIpsCounter.class).build();
1599         Optional<ExternalIpsCounter> externalCountersData;
1600         try {
1601             externalCountersData = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1602                         LogicalDatastoreType.OPERATIONAL, id);
1603         } catch (ReadFailedException e) {
1604             LOG.error("Failed to read external counters data for ExternalIp {}", externalIp, e);
1605             externalCountersData = Optional.absent();
1606         }
1607         if (externalCountersData.isPresent()) {
1608             ExternalIpsCounter externalIpsCounters = externalCountersData.get();
1609             for (ExternalCounters ext : externalIpsCounters.nonnullExternalCounters()) {
1610                 for (ExternalIpCounter externalIpCount : ext.nonnullExternalIpCounter()) {
1611                     if (externalIpCount.getExternalIp().equals(externalIp)) {
1612                         if (externalIpCount.getCounter() != 0) {
1613                             return true;
1614                         }
1615                         break;
1616                     }
1617                 }
1618             }
1619         }
1620         return false;
1621     }
1622
1623     private void allocateExternalIp(BigInteger dpnId, Routers router, long routerId, String routerName,
1624             Uuid networkId, String subnetIp, TypedWriteTransaction<Configuration> writeFlowInvTx) {
1625         String[] subnetIpParts = NatUtil.getSubnetIpAndPrefix(subnetIp);
1626         try {
1627             InetAddress address = InetAddress.getByName(subnetIpParts[0]);
1628             if (address instanceof Inet6Address) {
1629                 LOG.debug("allocateExternalIp : Skipping ipv6 address {} for the router {}.", address, routerName);
1630                 return;
1631             }
1632         } catch (UnknownHostException e) {
1633             LOG.error("allocateExternalIp : Invalid ip address {}", subnetIpParts[0], e);
1634             return;
1635         }
1636         String leastLoadedExtIpAddr = NatUtil.getLeastLoadedExternalIp(dataBroker, routerId);
1637         if (leastLoadedExtIpAddr != null) {
1638             String[] externalIpParts = NatUtil.getExternalIpAndPrefix(leastLoadedExtIpAddr);
1639             String leastLoadedExtIp = externalIpParts[0];
1640             String leastLoadedExtIpPrefix = externalIpParts[1];
1641             IPAddress externalIpAddr = new IPAddress(leastLoadedExtIp, Integer.parseInt(leastLoadedExtIpPrefix));
1642             subnetIp = subnetIpParts[0];
1643             String subnetIpPrefix = subnetIpParts[1];
1644             IPAddress subnetIpAddr = new IPAddress(subnetIp, Integer.parseInt(subnetIpPrefix));
1645             LOG.debug("allocateExternalIp : Add the IP mapping for the router ID {} and internal "
1646                     + "IP {} and prefix {} -> external IP {} and prefix {}",
1647                 routerId, subnetIp, subnetIpPrefix, leastLoadedExtIp, leastLoadedExtIpPrefix);
1648             naptManager.registerMapping(routerId, subnetIpAddr, externalIpAddr);
1649
1650
1651             // Check if external IP is already assigned a route. (i.e. External IP is previously
1652             // allocated to any of the subnets)
1653             // If external IP is already assigned a route, (, do not re-advertise to the BGP
1654             String leastLoadedExtIpAddrStr = leastLoadedExtIp + "/" + leastLoadedExtIpPrefix;
1655             Long label = checkExternalIpLabel(routerId, leastLoadedExtIpAddrStr);
1656             if (label != null) {
1657                 //update
1658                 String internalIp = subnetIpParts[0] + "/" + subnetIpParts[1];
1659                 IpMapKey ipMapKey = new IpMapKey(internalIp);
1660                 LOG.debug("allocateExternalIp : Setting label {} for internalIp {} and externalIp {}",
1661                     label, internalIp, leastLoadedExtIpAddrStr);
1662                 IpMap newIpm = new IpMapBuilder().withKey(ipMapKey).setInternalIp(internalIp)
1663                     .setExternalIp(leastLoadedExtIpAddrStr).setLabel(label).build();
1664                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
1665                     naptManager.getIpMapIdentifier(routerId, internalIp), newIpm);
1666                 return;
1667             }
1668
1669             // Re-advertise to the BGP for the external IP, which is allocated to the subnet
1670             // for the first time and hence not having a route.
1671             //Get the VPN Name using the network ID
1672             final String vpnName = NatUtil.getAssociatedVPN(dataBroker, networkId);
1673             if (vpnName != null) {
1674                 LOG.debug("allocateExternalIp : Retrieved vpnName {} for networkId {}", vpnName, networkId);
1675                 if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1676                     LOG.debug("allocateExternalIp : Best effort for getting primary napt switch when router i/f are"
1677                         + "added after gateway-set");
1678                     dpnId = NatUtil.getPrimaryNaptfromRouterId(dataBroker, routerId);
1679                     if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
1680                         LOG.error("allocateExternalIp : dpnId is null or Zero for the router {}", routerName);
1681                         return;
1682                     }
1683                 }
1684                 advToBgpAndInstallFibAndTsFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, vpnName, routerId, routerName,
1685                     leastLoadedExtIp + "/" + leastLoadedExtIpPrefix, networkId, router,
1686                     writeFlowInvTx);
1687             }
1688         }
1689     }
1690
1691     @Nullable
1692     protected Long checkExternalIpLabel(long routerId, String externalIp) {
1693         List<IpMap> ipMaps = naptManager.getIpMapList(dataBroker, routerId);
1694         for (IpMap ipMap : ipMaps) {
1695             if (ipMap.getExternalIp().equals(externalIp)) {
1696                 if (ipMap.getLabel() != null) {
1697                     return ipMap.getLabel();
1698                 }
1699             }
1700         }
1701         LOG.error("checkExternalIpLabel : no ipMaps found for routerID:{} and externalIP:{}", routerId, externalIp);
1702         return null;
1703     }
1704
1705     @Override
1706     protected void remove(InstanceIdentifier<Routers> identifier, Routers router) {
1707         LOG.trace("remove : Router delete method");
1708         /*
1709         ROUTER DELETE SCENARIO
1710         1) Get the router ID from the event.
1711         2) Build the cookie information from the router ID.
1712         3) Get the primary and secondary switch DPN IDs using the router ID from the model.
1713         4) Build the flow with the cookie value.
1714         5) Delete the flows which matches the cookie information from the NAPT outbound, inbound tables.
1715         6) Remove the flows from the other switches which points to the primary and secondary
1716          switches for the flows related the router ID.
1717         7) Get the list of external IP address maintained for the router ID.
1718         8) Use the NaptMananager removeMapping API to remove the list of IP addresses maintained.
1719         9) Withdraw the corresponding routes from the BGP.
1720          */
1721
1722         if (identifier == null || router == null) {
1723             LOG.error("remove : returning without processing since routers is null");
1724             return;
1725         }
1726
1727         String routerName = router.getRouterName();
1728         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.key(),
1729             () -> Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1730                 LOG.info("remove : Removing default NAT route from FIB on all dpns part of router {} ",
1731                         routerName);
1732                 Long routerId = NatUtil.getVpnId(dataBroker, routerName);
1733                 if (routerId == NatConstants.INVALID_ID) {
1734                     LOG.error("remove : Remove external router event - Invalid routerId for routerName {}",
1735                             routerName);
1736                     return;
1737                 }
1738                 long bgpVpnId = NatConstants.INVALID_ID;
1739                 Uuid bgpVpnUuid = NatUtil.getVpnForRouter(dataBroker, routerName);
1740                 if (bgpVpnUuid != null) {
1741                     bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnUuid.getValue());
1742                 }
1743                 addOrDelDefFibRouteToSNAT(routerName, routerId, bgpVpnId, bgpVpnUuid, false,
1744                         tx);
1745                 Uuid networkUuid = router.getNetworkId();
1746
1747                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
1748                 if (primarySwitchId == null || primarySwitchId.equals(BigInteger.ZERO)) {
1749                     // No NAPT switch for external router, probably because the router is not attached to
1750                     // any
1751                     // internal networks
1752                     LOG.debug(
1753                             "No NAPT switch for router {}, check if router is attached to any internal "
1754                                     + "network",
1755                             routerName);
1756                     return;
1757                 } else {
1758                     Collection<String> externalIps = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
1759                     handleDisableSnat(router, networkUuid, externalIps, true, null, primarySwitchId,
1760                             routerId, tx);
1761                 }
1762                 natOverVxlanUtil.releaseVNI(routerName);
1763             })), NatConstants.NAT_DJC_MAX_RETRIES);
1764     }
1765
1766     public void handleDisableSnat(Routers router, Uuid networkUuid, @NonNull Collection<String> externalIps,
1767                                   boolean routerFlag, @Nullable String vpnName, BigInteger naptSwitchDpnId,
1768                                   long routerId, TypedReadWriteTransaction<Configuration> removeFlowInvTx) {
1769         LOG.info("handleDisableSnat : Entry");
1770         String routerName = router.getRouterName();
1771         try {
1772             if (routerFlag) {
1773                 removeNaptSwitch(routerName);
1774             } else {
1775                 updateNaptSwitch(routerName, BigInteger.ZERO);
1776             }
1777
1778             LOG.debug("handleDisableSnat : Remove the ExternalCounter model for the router ID {}", routerId);
1779             naptManager.removeExternalCounter(routerId);
1780
1781             LOG.debug("handleDisableSnat : got primarySwitch as dpnId {}", naptSwitchDpnId);
1782             if (naptSwitchDpnId == null || naptSwitchDpnId.equals(BigInteger.ZERO)) {
1783                 LOG.error("handleDisableSnat : Unable to retrieve the primary NAPT switch for the "
1784                     + "router ID {} from RouterNaptSwitch model", routerId);
1785                 return;
1786             }
1787             ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName,
1788                     networkUuid);
1789             if (extNwProvType == null) {
1790                 LOG.error("handleDisableSnat : External Network Provider Type missing");
1791                 return;
1792             }
1793             Collection<Uuid> externalSubnetList = NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps());
1794             removeNaptFlowsFromActiveSwitch(routerId, routerName, naptSwitchDpnId, networkUuid, vpnName, externalIps,
1795                     externalSubnetList, removeFlowInvTx, extNwProvType);
1796             removeFlowsFromNonActiveSwitches(routerId, routerName, naptSwitchDpnId, removeFlowInvTx);
1797             String externalSubnetVpn = null;
1798             for (Uuid externalSubnetId : externalSubnetList) {
1799                 Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, externalSubnetId);
1800                 // externalSubnet data model will exist for FLAT/VLAN external netowrk UCs.
1801                 if (externalSubnet.isPresent()) {
1802                     externalSubnetVpn =  externalSubnetId.getValue();
1803                     clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, externalSubnetVpn,
1804                             router.getExtGwMacAddress(), removeFlowInvTx);
1805                 }
1806             }
1807             if (externalSubnetVpn == null) {
1808                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnName,
1809                         router.getExtGwMacAddress(), removeFlowInvTx);
1810             }
1811             // Use the NaptMananager removeMapping API to remove the entire list of IP addresses maintained
1812             // for the router ID.
1813             LOG.debug("handleDisableSnat : Remove the Internal to external IP address maintained for the "
1814                     + "router ID {} in the DS", routerId);
1815             naptManager.removeMapping(routerId);
1816         } catch (InterruptedException | ExecutionException e) {
1817             LOG.error("handleDisableSnat : Exception while handling disableSNAT for router :{}", routerName, e);
1818         }
1819         LOG.info("handleDisableSnat : Exit");
1820     }
1821
1822     // TODO Clean up the exception handling
1823     @SuppressWarnings("checkstyle:IllegalCatch")
1824     public void handleDisableSnatInternetVpn(String routerName, long routerId, Uuid networkUuid,
1825                                              @NonNull Collection<String> externalIps,
1826                                              String vpnId, TypedReadWriteTransaction<Configuration> writeFlowInvTx) {
1827         LOG.debug("handleDisableSnatInternetVpn: Started to process handle disable snat for router {} "
1828                 + "with internet vpn {}", routerName, vpnId);
1829         try {
1830             BigInteger naptSwitchDpnId = null;
1831             InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
1832                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
1833             Optional<RouterToNaptSwitch> rtrToNapt;
1834             try {
1835                 rtrToNapt = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1836                                 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
1837             } catch (ReadFailedException e) {
1838                 LOG.error("Failed to read NAPT switch for router {}", routerName, e);
1839                 rtrToNapt = Optional.absent();
1840             }
1841             if (rtrToNapt.isPresent()) {
1842                 naptSwitchDpnId = rtrToNapt.get().getPrimarySwitchId();
1843             }
1844             LOG.debug("handleDisableSnatInternetVpn : got primarySwitch as dpnId{} ", naptSwitchDpnId);
1845
1846             removeNaptFlowsFromActiveSwitchInternetVpn(routerId, routerName, naptSwitchDpnId, networkUuid, vpnId,
1847                     writeFlowInvTx);
1848             try {
1849                 String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
1850                 if (extGwMacAddress != null) {
1851                     LOG.debug("handleDisableSnatInternetVpn : External Gateway MAC address {} found for "
1852                             + "External Router ID {}", extGwMacAddress, routerId);
1853                 } else {
1854                     LOG.error("handleDisableSnatInternetVpn : No External Gateway MAC address found for "
1855                             + "External Router ID {}", routerId);
1856                     return;
1857                 }
1858                 clrRtsFromBgpAndDelFibTs(naptSwitchDpnId, routerId, networkUuid, externalIps, vpnId, extGwMacAddress,
1859                         writeFlowInvTx);
1860             } catch (Exception ex) {
1861                 LOG.error("handleDisableSnatInternetVpn : Failed to remove fib entries for routerId {} "
1862                         + "in naptSwitchDpnId {}", routerId, naptSwitchDpnId, ex);
1863             }
1864             natOverVxlanUtil.releaseVNI(vpnId);
1865         } catch (InterruptedException | ExecutionException e) {
1866             LOG.error("handleDisableSnatInternetVpn: Exception while handling disableSNATInternetVpn for router {} "
1867                     + "with internet vpn {}", routerName, vpnId, e);
1868         }
1869         LOG.debug("handleDisableSnatInternetVpn: Processed handle disable snat for router {} with internet vpn {}",
1870                 routerName, vpnId);
1871     }
1872
1873     // TODO Clean up the exception handling
1874     @SuppressWarnings("checkstyle:IllegalCatch")
1875     public void updateNaptSwitch(String routerName, BigInteger naptSwitchId) {
1876         RouterToNaptSwitch naptSwitch = new RouterToNaptSwitchBuilder().withKey(new RouterToNaptSwitchKey(routerName))
1877             .setPrimarySwitchId(naptSwitchId).build();
1878         try {
1879             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
1880                 NatUtil.buildNaptSwitchRouterIdentifier(routerName), naptSwitch);
1881         } catch (Exception ex) {
1882             LOG.error("updateNaptSwitch : Failed to write naptSwitch {} for router {} in ds",
1883                 naptSwitchId, routerName);
1884         }
1885         LOG.debug("updateNaptSwitch : Successfully updated naptSwitch {} for router {} in ds",
1886             naptSwitchId, routerName);
1887     }
1888
1889     protected void removeNaptSwitch(String routerName) {
1890         // Remove router and switch from model
1891         InstanceIdentifier<RouterToNaptSwitch> id =
1892             InstanceIdentifier.builder(NaptSwitches.class)
1893                 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
1894         LOG.debug("removeNaptSwitch : Removing NaptSwitch and Router for the router {} from datastore", routerName);
1895         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
1896         //Release allocated router_lPort_tag from the ID Manager if Router is on L3VPNOverVxlan
1897         NatEvpnUtil.releaseLPortTagForRouter(dataBroker, idManager, routerName);
1898     }
1899
1900     public void removeNaptFlowsFromActiveSwitch(long routerId, String routerName,
1901                                                 BigInteger dpnId, Uuid networkId, String vpnName,
1902                                                 @NonNull Collection<String> externalIps,
1903                                                 Collection<Uuid> externalSubnetList,
1904                                                 TypedReadWriteTransaction<Configuration> confTx,
1905                                                 ProviderTypes extNwProvType)
1906             throws InterruptedException, ExecutionException {
1907
1908         LOG.debug("removeNaptFlowsFromActiveSwitch : Remove NAPT flows from Active switch");
1909         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
1910
1911         //Remove the PSNAT entry which forwards the packet to Outbound NAPT Table (For the
1912         // traffic which comes from the  VMs of the NAPT switches)
1913         String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, routerName);
1914         FlowEntity preSnatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
1915
1916         LOG.info(
1917             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1918                 + "and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
1919         mdsalManager.removeFlow(confTx, preSnatFlowEntity);
1920
1921         //Remove the Terminating Service table entry which forwards the packet to Outbound NAPT Table (For the
1922         // traffic which comes from the VMs of the non NAPT switches)
1923         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
1924                 elanManager, idManager, routerId, routerName);
1925         String tsFlowRef = getFlowRefTs(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tunnelId);
1926         FlowEntity tsNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, tsFlowRef);
1927         LOG.info(
1928             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1929                 + "and router ID {}", NwConstants.INTERNAL_TUNNEL_TABLE, dpnId, routerId);
1930         mdsalManager.removeFlow(confTx, tsNatFlowEntity);
1931
1932         //Remove the flow table 25->44 from NAPT Switch
1933         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
1934             NatUtil.removePreDnatToSnatTableEntry(confTx, mdsalManager, dpnId);
1935         }
1936
1937         //Remove the Outbound flow entry which forwards the packet to FIB Table
1938         LOG.info(
1939             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {}"
1940                 + " and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
1941
1942         String outboundTcpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1943             NwConstants.IP_PROT_TCP);
1944         FlowEntity outboundTcpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1945             outboundTcpNatFlowRef);
1946         mdsalManager.removeFlow(confTx, outboundTcpNatFlowEntity);
1947
1948         String outboundUdpNatFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1949             NwConstants.IP_PROT_UDP);
1950         FlowEntity outboundUdpNatFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1951             outboundUdpNatFlowRef);
1952         mdsalManager.removeFlow(confTx, outboundUdpNatFlowEntity);
1953
1954         String icmpDropFlowRef = getFlowRefOutbound(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, routerId,
1955             NwConstants.IP_PROT_ICMP);
1956         FlowEntity icmpDropFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
1957             icmpDropFlowRef);
1958         mdsalManager.removeFlow(confTx, icmpDropFlowEntity);
1959         boolean lastRouterOnExternalNetwork =
1960             !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker, networkId, routerName, dpnId);
1961         if (lastRouterOnExternalNetwork) {
1962             removeNaptFibExternalOutputFlows(routerId, dpnId, networkId, externalIps, confTx);
1963         }
1964         //Remove the NAPT PFIB TABLE (47->21) which forwards the incoming packet to FIB Table matching on the
1965         // External Subnet Vpn Id.
1966         for (Uuid externalSubnetId : externalSubnetList) {
1967             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
1968             if (subnetVpnId != -1 && !NatUtil.checkForRoutersWithSameExtSubnetAndNaptSwitch(
1969                 dataBroker, externalSubnetId, routerName, dpnId)) {
1970                 String natPfibSubnetFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, subnetVpnId);
1971                 FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE,
1972                     natPfibSubnetFlowRef);
1973                 mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1974                 LOG.debug("removeNaptFlowsFromActiveSwitch : Removed the flow in table {} with external subnet "
1975                           + "Vpn Id {} as metadata on Napt Switch {}", NwConstants.NAPT_PFIB_TABLE,
1976                     subnetVpnId, dpnId);
1977             }
1978         }
1979
1980         //Remove the NAPT PFIB TABLE which forwards the incoming packet to FIB Table matching on the router ID.
1981         String natPfibFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, routerId);
1982         FlowEntity natPfibFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibFlowRef);
1983
1984         LOG.info(
1985             "removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch with the DPN ID {} "
1986             + "and router ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, routerId);
1987         mdsalManager.removeFlow(confTx, natPfibFlowEntity);
1988
1989         if (lastRouterOnExternalNetwork) {
1990             // Long vpnId = NatUtil.getVpnId(dataBroker, routerId);
1991             // - This does not work since ext-routers is deleted already - no network info
1992             //Get the VPN ID from the ExternalNetworks model
1993             long vpnId = -1;
1994             if (vpnName == null || vpnName.isEmpty()) {
1995                 // ie called from router delete cases
1996                 Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
1997                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnUuid is {}", vpnUuid);
1998                 if (vpnUuid != null) {
1999                     vpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2000                     LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId {} for external  network {} router delete "
2001                               + "or disableSNAT scenario", vpnId, networkId);
2002                 }
2003             } else {
2004                 // ie called from disassociate vpn case
2005                 LOG.debug("removeNaptFlowsFromActiveSwitch : This is disassociate nw with vpn case with vpnName {}",
2006                     vpnName);
2007                 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2008                 LOG.debug("removeNaptFlowsFromActiveSwitch : vpnId for disassociate nw with vpn scenario {}",
2009                     vpnId);
2010             }
2011
2012             if (vpnId != NatConstants.INVALID_ID) {
2013                 //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2014                 String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2015                 FlowEntity natPfibVpnFlowEntity =
2016                     NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2017                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in {} for the active switch with the "
2018                          + "DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2019                 mdsalManager.removeFlow(confTx, natPfibVpnFlowEntity);
2020             }
2021         }
2022
2023         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2024         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2025         if (ipPortMapping == null) {
2026             LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the IpPortMapping");
2027             return;
2028         }
2029
2030         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2031             String protocol = intextIpProtocolType.getProtocol().name();
2032             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2033                 String ipPortInternal = ipPortMap.getIpPortInternal();
2034                 String[] ipPortParts = ipPortInternal.split(":");
2035                 if (ipPortParts.length != 2) {
2036                     LOG.error("removeNaptFlowsFromActiveSwitch : Unable to retrieve the Internal IP and port");
2037                     return;
2038                 }
2039                 String internalIp = ipPortParts[0];
2040                 String internalPort = ipPortParts[1];
2041
2042                 //Build the flow for the outbound NAPT table
2043                 naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2044                     + NatConstants.COLON_SEPARATOR + internalPort);
2045                 String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2046                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2047                 FlowEntity outboundNaptFlowEntity =
2048                     NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2049
2050                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active switch "
2051                     + "with the DPN ID {} and router ID {}", NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2052                 mdsalManager.removeFlow(confTx, outboundNaptFlowEntity);
2053
2054                  //Build the flow for the inbound NAPT table
2055                 switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2056                     String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2057                 FlowEntity inboundNaptFlowEntity =
2058                     NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2059
2060                 LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in the {} for the active active switch "
2061                     + "with the DPN ID {} and router ID {}", NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2062                 mdsalManager.removeFlow(confTx, inboundNaptFlowEntity);
2063             }
2064         }
2065     }
2066
2067     protected void removeNaptFibExternalOutputFlows(long routerId, BigInteger dpnId, Uuid networkId,
2068                                                     @NonNull Collection<String> externalIps,
2069                                                     TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2070             throws ExecutionException, InterruptedException {
2071         long extVpnId = NatConstants.INVALID_ID;
2072         if (networkId != null) {
2073             Uuid vpnUuid = NatUtil.getVpnIdfromNetworkId(dataBroker, networkId);
2074             if (vpnUuid != null) {
2075                 extVpnId = NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
2076             } else {
2077                 LOG.debug("removeNaptFibExternalOutputFlows : vpnUuid is null");
2078             }
2079         } else {
2080             LOG.debug("removeNaptFibExternalOutputFlows : networkId is null");
2081             extVpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2082         }
2083         if (extVpnId == NatConstants.INVALID_ID) {
2084             LOG.warn("removeNaptFibExternalOutputFlows : extVpnId not found for routerId {}", routerId);
2085             extVpnId = routerId;
2086         }
2087         for (String ip : externalIps) {
2088             String extIp = removeMaskFromIp(ip);
2089             String naptFlowRef = getFlowRefNaptPreFib(dpnId, NwConstants.NAPT_PFIB_TABLE, extVpnId);
2090             LOG.info("removeNaptFlowsFromActiveSwitch : Remove the flow in table {} for the active switch"
2091                 + " with the DPN ID {} and router ID {} and IP {} flowRef {}",
2092                 NwConstants.NAPT_PFIB_TABLE, dpnId, routerId, extIp, naptFlowRef);
2093             FlowEntity natPfibVpnFlowEntity = NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, naptFlowRef);
2094             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2095         }
2096     }
2097
2098     private String removeMaskFromIp(String ip) {
2099         if (ip != null && !ip.trim().isEmpty()) {
2100             return ip.split("/")[0];
2101         }
2102         return ip;
2103     }
2104
2105     public void removeNaptFlowsFromActiveSwitchInternetVpn(long routerId, String routerName,
2106                                                            BigInteger dpnId, Uuid networkId, String vpnName,
2107                                                            TypedReadWriteTransaction<Configuration> writeFlowInvTx)
2108             throws ExecutionException, InterruptedException {
2109         LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : Remove NAPT flows from Active switch Internet Vpn");
2110         BigInteger cookieSnatFlow = NatUtil.getCookieNaptFlow(routerId);
2111
2112         //Remove the NAPT PFIB TABLE entry
2113         long vpnId = -1;
2114         if (vpnName != null) {
2115             // ie called from disassociate vpn case
2116             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : This is disassociate nw with vpn case "
2117                     + "with vpnName {}", vpnName);
2118             vpnId = NatUtil.getVpnId(dataBroker, vpnName);
2119             LOG.debug("removeNaptFlowsFromActiveSwitchInternetVpn : vpnId for disassociate nw with vpn scenario {}",
2120                     vpnId);
2121         }
2122
2123         if (vpnId != NatConstants.INVALID_ID && !NatUtil.checkForRoutersWithSameExtNetAndNaptSwitch(dataBroker,
2124                 networkId, routerName, dpnId)) {
2125             //Remove the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2126             String natPfibVpnFlowRef = getFlowRefTs(dpnId, NwConstants.NAPT_PFIB_TABLE, vpnId);
2127             FlowEntity natPfibVpnFlowEntity =
2128                 NatUtil.buildFlowEntity(dpnId, NwConstants.NAPT_PFIB_TABLE, natPfibVpnFlowRef);
2129             LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the active switch "
2130                     + "with the DPN ID {} and VPN ID {}", NwConstants.NAPT_PFIB_TABLE, dpnId, vpnId);
2131             mdsalManager.removeFlow(writeFlowInvTx, natPfibVpnFlowEntity);
2132
2133             // Remove IP-PORT active NAPT entries and release port from IdManager
2134             // For the router ID get the internal IP , internal port and the corresponding
2135             // external IP and external Port.
2136             IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2137             if (ipPortMapping == null) {
2138                 LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the IpPortMapping");
2139                 return;
2140             }
2141             for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2142                 String protocol = intextIpProtocolType.getProtocol().name();
2143                 for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2144                     String ipPortInternal = ipPortMap.getIpPortInternal();
2145                     String[] ipPortParts = ipPortInternal.split(":");
2146                     if (ipPortParts.length != 2) {
2147                         LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Unable to retrieve the Internal IP "
2148                                 + "and port");
2149                         return;
2150                     }
2151                     String internalIp = ipPortParts[0];
2152                     String internalPort = ipPortParts[1];
2153
2154                     //Build the flow for the outbound NAPT table
2155                     naptPacketInHandler.removeIncomingPacketMap(routerId + NatConstants.COLON_SEPARATOR + internalIp
2156                             + NatConstants.COLON_SEPARATOR + internalPort);
2157                     String switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.OUTBOUND_NAPT_TABLE,
2158                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2159                     FlowEntity outboundNaptFlowEntity =
2160                         NatUtil.buildFlowEntity(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2161
2162                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2163                             + "active switch with the DPN ID {} and router ID {}",
2164                             NwConstants.OUTBOUND_NAPT_TABLE, dpnId, routerId);
2165                     mdsalManager.removeFlow(writeFlowInvTx, outboundNaptFlowEntity);
2166
2167                     IpPortExternal ipPortExternal = ipPortMap.getIpPortExternal();
2168                     final String externalIp = ipPortExternal.getIpAddress();
2169
2170                     //Build the flow for the inbound NAPT table
2171                     switchFlowRef = NatUtil.getNaptFlowRef(dpnId, NwConstants.INBOUND_NAPT_TABLE,
2172                         String.valueOf(routerId), internalIp, Integer.parseInt(internalPort), protocol);
2173                     FlowEntity inboundNaptFlowEntity =
2174                         NatUtil.buildFlowEntity(dpnId, NwConstants.INBOUND_NAPT_TABLE, cookieSnatFlow, switchFlowRef);
2175
2176                     LOG.info("removeNaptFlowsFromActiveSwitchInternetVpn : Remove the flow in the {} for the "
2177                             + "active active switch with the DPN ID {} and router ID {}",
2178                             NwConstants.INBOUND_NAPT_TABLE, dpnId, routerId);
2179                     mdsalManager.removeFlow(writeFlowInvTx, inboundNaptFlowEntity);
2180
2181                     // Finally release port from idmanager
2182                     String internalIpPort = internalIp + ":" + internalPort;
2183                     naptManager.removePortFromPool(internalIpPort, externalIp);
2184
2185                     //Remove sessions from models
2186                     naptManager.removeIpPortMappingForRouterID(routerId);
2187                     naptManager.removeIntIpPortMappingForRouterID(routerId);
2188                 }
2189             }
2190         } else {
2191             LOG.error("removeNaptFlowsFromActiveSwitchInternetVpn : Invalid vpnId {}", vpnId);
2192         }
2193     }
2194
2195     public void removeFlowsFromNonActiveSwitches(long routerId, String routerName,
2196             BigInteger naptSwitchDpnId, TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2197             throws ExecutionException, InterruptedException {
2198         LOG.debug("removeFlowsFromNonActiveSwitches : Remove NAPT related flows from non active switches");
2199
2200         // Remove the flows from the other switches which points to the primary and secondary switches
2201         // for the flows related the router ID.
2202         List<BigInteger> allSwitchList = naptSwitchSelector.getDpnsForVpn(routerName);
2203         if (allSwitchList.isEmpty()) {
2204             LOG.error("removeFlowsFromNonActiveSwitches : Unable to get the swithces for the router {}", routerName);
2205             return;
2206         }
2207         for (BigInteger dpnId : allSwitchList) {
2208             if (!naptSwitchDpnId.equals(dpnId)) {
2209                 LOG.info("removeFlowsFromNonActiveSwitches : Handle Ordinary switch");
2210
2211                 //Remove the PSNAT entry which forwards the packet to Terminating Service table
2212                 String preSnatFlowRef = getFlowRefSnat(dpnId, NwConstants.PSNAT_TABLE, String.valueOf(routerName));
2213                 FlowEntity preSnatFlowEntity =
2214                     NatUtil.buildFlowEntity(dpnId, NwConstants.PSNAT_TABLE, preSnatFlowRef);
2215
2216                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the flow in the {} for the non active switch "
2217                     + "with the DPN ID {} and router ID {}", NwConstants.PSNAT_TABLE, dpnId, routerId);
2218                 mdsalManager.removeFlow(removeFlowInvTx, preSnatFlowEntity);
2219
2220                 //Remove the group entry which forwards the traffic to the out port (VXLAN tunnel).
2221                 long groupId = createGroupId(getGroupIdKey(routerName));
2222
2223                 LOG.info("removeFlowsFromNonActiveSwitches : Remove the group {} for the non active switch with "
2224                     + "the DPN ID {} and router ID {}", groupId, dpnId, routerId);
2225                 mdsalManager.removeGroup(removeFlowInvTx, dpnId, groupId);
2226             }
2227         }
2228     }
2229
2230     public void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, Long routerId, @Nullable Uuid networkUuid,
2231                                          @NonNull Collection<String> externalIps, @Nullable String vpnName,
2232                                          String extGwMacAddress, TypedReadWriteTransaction<Configuration> confTx)
2233             throws ExecutionException, InterruptedException {
2234         //Withdraw the corresponding routes from the BGP.
2235         //Get the network ID using the router ID.
2236         LOG.debug("clrRtsFromBgpAndDelFibTs : Advertise to BGP and remove routes for externalIps {} with routerId {},"
2237                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2238         if (networkUuid == null) {
2239             LOG.error("clrRtsFromBgpAndDelFibTs : networkId is null");
2240             return;
2241         }
2242
2243         if (externalIps.isEmpty()) {
2244             LOG.error("clrRtsFromBgpAndDelFibTs : externalIps is empty");
2245             return;
2246         }
2247
2248         if (vpnName == null) {
2249             //Get the VPN Name using the network ID
2250             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2251             if (vpnName == null) {
2252                 LOG.error("clrRtsFromBgpAndDelFibTs : No VPN associated with ext nw {} for the router {}",
2253                     networkUuid, routerId);
2254                 return;
2255             }
2256         }
2257         LOG.debug("clrRtsFromBgpAndDelFibTs : Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2258
2259         //Remove custom FIB routes
2260         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2261         for (String extIp : externalIps) {
2262             clrRtsFromBgpAndDelFibTs(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, confTx);
2263         }
2264     }
2265
2266     protected void clrRtsFromBgpAndDelFibTs(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2267                                             final Uuid networkUuid, String extGwMacAddress,
2268                                             TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2269             throws ExecutionException, InterruptedException {
2270         clearBgpRoutes(extIp, vpnName);
2271         delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2272                 removeFlowInvTx);
2273     }
2274
2275     protected void delFibTsAndReverseTraffic(final BigInteger dpnId, String routerName, long routerId, String extIp,
2276                                              String vpnName, Uuid extNetworkId, long tempLabel,
2277                                              String gwMacAddress, boolean switchOver,
2278                                              TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2279             throws ExecutionException, InterruptedException {
2280         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2281         //String routerName = NatUtil.getRouterName(dataBroker,routerId);
2282         if (routerName == null) {
2283             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2284             return;
2285         }
2286         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
2287         if (extNwProvType == null) {
2288             LOG.error("delFibTsAndReverseTraffic : External Network Provider Type Missing");
2289             return;
2290         }
2291         /*  Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2292          * external network provided type is VxLAN
2293          */
2294         if (extNwProvType == ProviderTypes.VXLAN) {
2295             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, gwMacAddress
2296             );
2297             return;
2298         }
2299         if (tempLabel < 0) {
2300             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2301                     extIp, routerId);
2302             return;
2303         }
2304         final long label = tempLabel;
2305         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2306         if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
2307             LOG.debug("delFibTsAndReverseTraffic : Using extSubnetId as vpnName for FLAT/VLAN use-cases");
2308             Routers extRouter = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
2309             Uuid externalSubnetId = NatUtil.getExternalSubnetForRouterExternalIp(externalIp,
2310                     extRouter);
2311
2312             Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker,
2313                     externalSubnetId);
2314
2315             if (externalSubnet.isPresent()) {
2316                 vpnName =  externalSubnetId.getValue();
2317             }
2318         }
2319         final String externalVpn = vpnName;
2320         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(externalVpn)
2321                 .setSourceDpid(dpnId).setIpAddress(externalIp).setServiceId(label)
2322                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).build();
2323         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2324
2325         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2326         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2327         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2328             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2329             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2330         }
2331         if (!switchOver) {
2332             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2333                 Futures.transformAsync(future, result -> {
2334                     //Release label
2335                     if (result.isSuccessful()) {
2336                         NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2337                         RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2338                             .setVpnName(externalVpn).setIpPrefix(externalIp).build();
2339                         return vpnService.removeVpnLabel(labelInput);
2340                     } else {
2341                         String errMsg =
2342                             String.format("RPC call to remove custom FIB entries on dpn %s for "
2343                                 + "prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
2344                         LOG.error(errMsg);
2345                         return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2346                     }
2347                 }, MoreExecutors.directExecutor());
2348
2349             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2350
2351                 @Override
2352                 public void onFailure(@NonNull Throwable error) {
2353                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label:{} or custom fib entries"
2354                         + "got external ip {}", label, extIp, error);
2355                 }
2356
2357                 @Override
2358                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2359                     if (result.isSuccessful()) {
2360                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2361                             + "from VPN {}", externalIp, externalVpn);
2362                     } else {
2363                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2364                             + " from VPN {}, {}", externalIp, externalVpn, result.getErrors());
2365                     }
2366                 }
2367             }, MoreExecutors.directExecutor());
2368         } else {
2369             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2370                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2371         }
2372     }
2373
2374     private void delFibTsAndReverseTraffic(final BigInteger dpnId, long routerId, String extIp, final String vpnName,
2375                                            final Uuid networkUuid, String extGwMacAddress, boolean switchOver,
2376                                            TypedReadWriteTransaction<Configuration> removeFlowInvTx)
2377             throws ExecutionException, InterruptedException {
2378         LOG.debug("delFibTsAndReverseTraffic : Removing fib entry for externalIp {} in routerId {}", extIp, routerId);
2379         String routerName = NatUtil.getRouterName(dataBroker,routerId);
2380         if (routerName == null) {
2381             LOG.error("delFibTsAndReverseTraffic : Could not retrieve Router Name from Router ID {} ", routerId);
2382             return;
2383         }
2384         //Get the external network provider type from networkId
2385         ProviderTypes extNwProvType = NatUtil.getProviderTypefromNetworkId(dataBroker, networkUuid);
2386         if (extNwProvType == null) {
2387             LOG.error("delFibTsAndReverseTraffic : Could not retrieve provider type for external network {} ",
2388                     networkUuid);
2389             return;
2390         }
2391         /* Remove the flow table19->44 and table36->44 entries for SNAT reverse traffic flow if the
2392          *  external network provided type is VxLAN
2393          */
2394         if (extNwProvType == ProviderTypes.VXLAN) {
2395             evpnSnatFlowProgrammer.evpnDelFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, extGwMacAddress);
2396             return;
2397         }
2398         //Get IPMaps from the DB for the router ID
2399         List<IpMap> dbIpMaps = NaptManager.getIpMapList(dataBroker, routerId);
2400         if (dbIpMaps.isEmpty()) {
2401             LOG.error("delFibTsAndReverseTraffic : IPMaps not found for router {}", routerId);
2402             return;
2403         }
2404
2405         long tempLabel = NatConstants.INVALID_ID;
2406         for (IpMap dbIpMap : dbIpMaps) {
2407             String dbExternalIp = dbIpMap.getExternalIp();
2408             LOG.debug("delFibTsAndReverseTraffic : Retrieved dbExternalIp {} for router id {}", dbExternalIp, routerId);
2409             //Select the IPMap, whose external IP is the IP for which FIB is installed
2410             if (extIp.equals(dbExternalIp)) {
2411                 tempLabel = dbIpMap.getLabel();
2412                 LOG.debug("delFibTsAndReverseTraffic : Retrieved label {} for dbExternalIp {} with router id {}",
2413                     tempLabel, dbExternalIp, routerId);
2414                 break;
2415             }
2416         }
2417         if (tempLabel == NatConstants.INVALID_ID) {
2418             LOG.error("delFibTsAndReverseTraffic : Label not found for externalIp {} with router id {}",
2419                     extIp, routerId);
2420             return;
2421         }
2422
2423         final long label = tempLabel;
2424         final String externalIp = NatUtil.validateAndAddNetworkMask(extIp);
2425         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder()
2426                 .setVpnName(vpnName).setSourceDpid(dpnId).setIpAddress(externalIp)
2427                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.ExternalFixedIP).setServiceId(label).build();
2428         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
2429
2430         removeTunnelTableEntry(dpnId, label, removeFlowInvTx);
2431         removeLFibTableEntry(dpnId, label, removeFlowInvTx);
2432         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2433             //Remove the flow table 25->44 If there is no FIP Match on table 25 (PDNAT_TABLE)
2434             NatUtil.removePreDnatToSnatTableEntry(removeFlowInvTx, mdsalManager, dpnId);
2435         }
2436         if (!switchOver) {
2437             ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture =
2438                     Futures.transformAsync(future, result -> {
2439                         //Release label
2440                         if (result.isSuccessful()) {
2441                             RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
2442                                     .setVpnName(vpnName).setIpPrefix(externalIp).build();
2443                             return vpnService.removeVpnLabel(labelInput);
2444                         } else {
2445                             String errMsg =
2446                                     String.format("RPC call to remove custom FIB entries on dpn %s for "
2447                                             + "prefix %s Failed - %s",
2448                                             dpnId, externalIp, result.getErrors());
2449                             LOG.error(errMsg);
2450                             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
2451                         }
2452                     }, MoreExecutors.directExecutor());
2453
2454             Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
2455
2456                 @Override
2457                 public void onFailure(@NonNull Throwable error) {
2458                     LOG.error("delFibTsAndReverseTraffic : Error in removing the label or custom fib entries", error);
2459                 }
2460
2461                 @Override
2462                 public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
2463                     if (result.isSuccessful()) {
2464                         LOG.debug("delFibTsAndReverseTraffic : Successfully removed the label for the prefix {} "
2465                             + "from VPN {}", externalIp, vpnName);
2466                     } else {
2467                         LOG.error("delFibTsAndReverseTraffic : Error in removing the label for prefix {} "
2468                             + " from VPN {}, {}", externalIp, vpnName, result.getErrors());
2469                     }
2470                 }
2471             }, MoreExecutors.directExecutor());
2472         } else {
2473             LOG.debug("delFibTsAndReverseTraffic: switch-over is happened on DpnId {}. No need to release allocated "
2474                     + "label {} for external fixed ip {} for router {}", dpnId, label, externalIp, routerId);
2475         }
2476     }
2477
2478     protected void clearFibTsAndReverseTraffic(final BigInteger dpnId, Long routerId, Uuid networkUuid,
2479             List<String> externalIps, @Nullable String vpnName, String extGwMacAddress,
2480             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2481         //Withdraw the corresponding routes from the BGP.
2482         //Get the network ID using the router ID.
2483         LOG.debug("clearFibTsAndReverseTraffic : for externalIps {} with routerId {},"
2484                 + "network Id {} and vpnName {}", externalIps, routerId, networkUuid, vpnName);
2485         if (networkUuid == null) {
2486             LOG.error("clearFibTsAndReverseTraffic : networkId is null");
2487             return;
2488         }
2489
2490         if (externalIps == null || externalIps.isEmpty()) {
2491             LOG.error("clearFibTsAndReverseTraffic : externalIps is null");
2492             return;
2493         }
2494
2495         if (vpnName == null) {
2496             //Get the VPN Name using the network ID
2497             vpnName = NatUtil.getAssociatedVPN(dataBroker, networkUuid);
2498             if (vpnName == null) {
2499                 LOG.error("clearFibTsAndReverseTraffic : No VPN associated with ext nw {} for the router {}",
2500                     networkUuid, routerId);
2501                 return;
2502             }
2503         }
2504         LOG.debug("Retrieved vpnName {} for networkId {}", vpnName, networkUuid);
2505
2506         //Remove custom FIB routes
2507         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
2508         for (String extIp : externalIps) {
2509             delFibTsAndReverseTraffic(dpnId, routerId, extIp, vpnName, networkUuid, extGwMacAddress, false,
2510                     writeFlowInvTx);
2511         }
2512     }
2513
2514     protected void clearBgpRoutes(String externalIp, final String vpnName) {
2515         //Inform BGP about the route removal
2516         LOG.info("clearBgpRoutes : Informing BGP to remove route for externalIP {} of vpn {}", externalIp, vpnName);
2517         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
2518         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, externalIp, vpnName, LOG);
2519     }
2520
2521     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
2522             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2523         LOG.info("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
2524         mdsalManager.removeFlow(writeFlowInvTx, dpnId,
2525             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), NwConstants.INTERNAL_TUNNEL_TABLE);
2526         LOG.debug("removeTunnelTableEntry : dpID {} : label : {} removed successfully", dpnId, serviceId);
2527     }
2528
2529     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
2530             TypedReadWriteTransaction<Configuration> writeFlowInvTx) throws ExecutionException, InterruptedException {
2531         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
2532         LOG.debug("removeLFibTableEntry : with flow ref {}", flowRef);
2533         mdsalManager.removeFlow(writeFlowInvTx, dpnId, flowRef, NwConstants.L3_LFIB_TABLE);
2534         LOG.debug("removeLFibTableEntry : dpID : {} label : {} removed successfully", dpnId, serviceId);
2535     }
2536
2537     /**
2538      * router association to vpn.
2539      *
2540      * @param routerName - Name of router
2541      * @param routerId - router id
2542      * @param bgpVpnName BGP VPN name
2543      */
2544     public void changeLocalVpnIdToBgpVpnId(String routerName, long routerId, String bgpVpnName,
2545             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2546         LOG.debug("changeLocalVpnIdToBgpVpnId : Router associated to BGP VPN");
2547         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2548             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2549
2550             LOG.debug("changeLocalVpnIdToBgpVpnId : BGP VPN ID value {} ", bgpVpnId);
2551
2552             if (bgpVpnId != NatConstants.INVALID_ID) {
2553                 LOG.debug("changeLocalVpnIdToBgpVpnId : Populate the router-id-name container with the "
2554                         + "mapping BGP VPN-ID {} -> BGP VPN-NAME {}", bgpVpnId, bgpVpnName);
2555                 RouterIds rtrs = new RouterIdsBuilder().withKey(new RouterIdsKey(bgpVpnId))
2556                     .setRouterId(bgpVpnId).setRouterName(bgpVpnName).build();
2557                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
2558                     getRoutersIdentifier(bgpVpnId), rtrs);
2559
2560                 // Get the allocated Primary NAPT Switch for this router
2561                 LOG.debug("changeLocalVpnIdToBgpVpnId : Router ID value {} ", routerId);
2562
2563                 LOG.debug("changeLocalVpnIdToBgpVpnId : Update the Router ID {} to the BGP VPN ID {} ",
2564                         routerId, bgpVpnId);
2565                 addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, bgpVpnId, writeFlowInvTx);
2566
2567                 // Get the group ID
2568                 BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2569                 createGroupId(getGroupIdKey(routerName));
2570                 installFlowsWithUpdatedVpnId(primarySwitchId, routerName, bgpVpnId, routerId, true, writeFlowInvTx,
2571                         extNwProvType);
2572             }
2573         }
2574     }
2575
2576     /**
2577      * router disassociation from vpn.
2578      *
2579      * @param routerName - Name of router
2580      * @param routerId - router id
2581      * @param bgpVpnName BGP VPN name
2582      */
2583     public void changeBgpVpnIdToLocalVpnId(String routerName, long routerId, String bgpVpnName,
2584             TypedWriteTransaction<Configuration> writeFlowInvTx, ProviderTypes extNwProvType) {
2585         LOG.debug("changeBgpVpnIdToLocalVpnId : Router dissociated from BGP VPN");
2586         if (chkExtRtrAndSnatEnbl(new Uuid(routerName))) {
2587             long bgpVpnId = NatUtil.getVpnId(dataBroker, bgpVpnName);
2588             LOG.debug("changeBgpVpnIdToLocalVpnId : BGP VPN ID value {} ", bgpVpnId);
2589
2590             // Get the allocated Primary NAPT Switch for this router
2591             LOG.debug("changeBgpVpnIdToLocalVpnId : Router ID value {} ", routerId);
2592
2593             LOG.debug("changeBgpVpnIdToLocalVpnId : Update the BGP VPN ID {} to the Router ID {}", bgpVpnId, routerId);
2594             addDefaultFibRouteForSnatWithBgpVpn(routerName, routerId, NatConstants.INVALID_ID, writeFlowInvTx);
2595
2596             // Get the group ID
2597             createGroupId(getGroupIdKey(routerName));
2598             BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
2599             installFlowsWithUpdatedVpnId(primarySwitchId, routerName, NatConstants.INVALID_ID, routerId, true,
2600                     writeFlowInvTx, extNwProvType);
2601         }
2602     }
2603
2604     boolean chkExtRtrAndSnatEnbl(Uuid routerUuid) {
2605         InstanceIdentifier<Routers> routerInstanceIndentifier =
2606             InstanceIdentifier.builder(ExtRouters.class)
2607                 .child(Routers.class, new RoutersKey(routerUuid.getValue())).build();
2608         try {
2609             Optional<Routers> routerData = SingleTransactionDataBroker
2610                     .syncReadOptional(dataBroker,
2611                             LogicalDatastoreType.CONFIGURATION, routerInstanceIndentifier);
2612             return routerData.isPresent() && routerData.get().isEnableSnat();
2613         } catch (ReadFailedException e) {
2614             LOG.error("Failed to read data for router id {}", routerUuid, e);
2615             return false;
2616         }
2617     }
2618
2619     public void installFlowsWithUpdatedVpnId(BigInteger primarySwitchId, String routerName, long bgpVpnId,
2620         long routerId, boolean isSnatCfgd, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2621
2622         long changedVpnId = bgpVpnId;
2623         String idType = "BGP VPN";
2624         if (bgpVpnId == NatConstants.INVALID_ID) {
2625             changedVpnId = routerId;
2626             idType = "router";
2627         }
2628
2629         List<BigInteger> switches = NatUtil.getDpnsForRouter(dataBroker, routerName);
2630         if (switches.isEmpty()) {
2631             LOG.error("installFlowsWithUpdatedVpnId : No switches found for router {}", routerName);
2632             return;
2633         }
2634         for (BigInteger dpnId : switches) {
2635             // Update the BGP VPN ID in the SNAT miss entry to group
2636             if (!dpnId.equals(primarySwitchId)) {
2637                 LOG.debug("installFlowsWithUpdatedVpnId : Install group in non NAPT switch {}", dpnId);
2638                 List<BucketInfo> bucketInfoForNonNaptSwitches =
2639                     getBucketInfoForNonNaptSwitches(dpnId, primarySwitchId, routerName, routerId);
2640                 long groupId = createGroupId(getGroupIdKey(routerName));
2641                 if (!isSnatCfgd) {
2642                     groupId = installGroup(dpnId, routerName, bucketInfoForNonNaptSwitches);
2643                 }
2644
2645                 LOG.debug(
2646                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2647                                 + "{} in the non NAPT switch {}", idType, changedVpnId, groupId, dpnId);
2648                 FlowEntity flowEntity = buildSnatFlowEntityWithUpdatedVpnId(dpnId, routerName, groupId, changedVpnId);
2649                 mdsalManager.addFlow(confTx, flowEntity);
2650             } else {
2651                 LOG.debug(
2652                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the SNAT miss entry pointing to group "
2653                                 + "in the primary switch {}", idType, changedVpnId, primarySwitchId);
2654                 FlowEntity flowEntity =
2655                     buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(primarySwitchId, routerName, changedVpnId);
2656                 mdsalManager.addFlow(confTx, flowEntity);
2657
2658                 LOG.debug(
2659                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Terminating Service table (table "
2660                                 + "ID 36) which forwards the packet to the table 46 in the Primary switch {}",
2661                         idType, changedVpnId, primarySwitchId);
2662                 installTerminatingServiceTblEntryWithUpdatedVpnId(primarySwitchId, routerName, routerId,
2663                         changedVpnId, confTx, extNwProvType);
2664
2665                 LOG.debug(
2666                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the Outbound NAPT table (table ID 46) "
2667                                 + "which punts the packet to the controller in the Primary switch {}",
2668                         idType, changedVpnId, primarySwitchId);
2669                 createOutboundTblEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2670
2671                 LOG.debug(
2672                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT PFIB TABLE which forwards the"
2673                                 + " outgoing packet to FIB Table in the Primary switch {}",
2674                         idType, changedVpnId, primarySwitchId);
2675                 installNaptPfibEntryWithBgpVpn(primarySwitchId, routerId, changedVpnId, confTx);
2676
2677                 LOG.debug(
2678                         "installFlowsWithUpdatedVpnId : Update the {} ID {} in the NAPT flows for the Outbound NAPT "
2679                                 + "table (table ID 46) and the INBOUND NAPT table (table ID 44) in the Primary switch"
2680                                 + " {}", idType, changedVpnId, primarySwitchId);
2681                 updateNaptFlowsWithVpnId(primarySwitchId, routerName, routerId, bgpVpnId);
2682
2683                 LOG.debug("installFlowsWithUpdatedVpnId : Installing SNAT PFIB flow in the primary switch {}",
2684                         primarySwitchId);
2685                 Long vpnId = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2686                 //Install the NAPT PFIB TABLE which forwards the outgoing packet to FIB Table matching on the VPN ID.
2687                 if (vpnId != NatConstants.INVALID_ID) {
2688                     installNaptPfibEntry(primarySwitchId, vpnId, confTx);
2689                 }
2690             }
2691         }
2692     }
2693
2694     public void updateNaptFlowsWithVpnId(BigInteger dpnId, String routerName, long routerId, long bgpVpnId) {
2695         //For the router ID get the internal IP , internal port and the corresponding external IP and external Port.
2696         IpPortMapping ipPortMapping = NatUtil.getIportMapping(dataBroker, routerId);
2697         if (ipPortMapping == null) {
2698             LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the IpPortMapping");
2699             return;
2700         }
2701         // Get the External Gateway MAC Address
2702         String extGwMacAddress = NatUtil.getExtGwMacAddFromRouterName(dataBroker, routerName);
2703         if (extGwMacAddress != null) {
2704             LOG.debug("updateNaptFlowsWithVpnId : External Gateway MAC address {} found for External Router ID {}",
2705                     extGwMacAddress, routerId);
2706         } else {
2707             LOG.error("updateNaptFlowsWithVpnId : No External Gateway MAC address found for External Router ID {}",
2708                     routerId);
2709             return;
2710         }
2711         for (IntextIpProtocolType intextIpProtocolType : ipPortMapping.nonnullIntextIpProtocolType()) {
2712             for (IpPortMap ipPortMap : intextIpProtocolType.nonnullIpPortMap()) {
2713                 String ipPortInternal = ipPortMap.getIpPortInternal();
2714                 String[] ipPortParts = ipPortInternal.split(":");
2715                 if (ipPortParts.length != 2) {
2716                     LOG.error("updateNaptFlowsWithVpnId : Unable to retrieve the Internal IP and port");
2717                     return;
2718                 }
2719                 String internalIp = ipPortParts[0];
2720                 String internalPort = ipPortParts[1];
2721                 LOG.debug("updateNaptFlowsWithVpnId : Found Internal IP {} and Internal Port {}",
2722                         internalIp, internalPort);
2723                 ProtocolTypes protocolTypes = intextIpProtocolType.getProtocol();
2724                 NAPTEntryEvent.Protocol protocol;
2725                 switch (protocolTypes) {
2726                     case TCP:
2727                         protocol = NAPTEntryEvent.Protocol.TCP;
2728                         break;
2729                     case UDP:
2730                         protocol = NAPTEntryEvent.Protocol.UDP;
2731                         break;
2732                     default:
2733                         protocol = NAPTEntryEvent.Protocol.TCP;
2734                 }
2735                 SessionAddress internalAddress = new SessionAddress(internalIp, Integer.parseInt(internalPort));
2736                 SessionAddress externalAddress =
2737                         naptManager.getExternalAddressMapping(routerId, internalAddress, protocol);
2738                 long internetVpnid = NatUtil.getNetworkVpnIdFromRouterId(dataBroker, routerId);
2739                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.INBOUND_NAPT_TABLE, internetVpnid,
2740                         routerId, bgpVpnId, externalAddress, internalAddress, protocol, extGwMacAddress);
2741                 naptEventHandler.buildAndInstallNatFlows(dpnId, NwConstants.OUTBOUND_NAPT_TABLE, internetVpnid,
2742                         routerId, bgpVpnId, internalAddress, externalAddress, protocol, extGwMacAddress);
2743             }
2744         }
2745     }
2746
2747     public FlowEntity buildSnatFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName, long groupId,
2748                                                           long changedVpnId) {
2749
2750         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : called for dpId {}, routerName {} groupId {} "
2751             + "changed VPN ID {}", dpId, routerName, groupId, changedVpnId);
2752         List<MatchInfo> matches = new ArrayList<>();
2753         matches.add(MatchEthernetType.IPV4);
2754         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2755
2756         List<ActionInfo> actionsInfo = new ArrayList<>();
2757         long tunnelId = NatUtil.getTunnelIdForNonNaptToNaptFlow(dataBroker, natOverVxlanUtil,
2758                 elanManager, idManager, changedVpnId, routerName);
2759         actionsInfo.add(new ActionSetFieldTunnelId(BigInteger.valueOf(tunnelId)));
2760         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Setting the tunnel to the list of action infos {}",
2761                 actionsInfo);
2762         actionsInfo.add(new ActionGroup(groupId));
2763         List<InstructionInfo> instructions = new ArrayList<>();
2764         instructions.add(new InstructionApplyActions(actionsInfo));
2765         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2766         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2767             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2768             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2769
2770         LOG.debug("buildSnatFlowEntityWithUpdatedVpnId : Returning SNAT Flow Entity {}", flowEntity);
2771         return flowEntity;
2772     }
2773
2774     public FlowEntity buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch(BigInteger dpId, String routerName,
2775                                                                         long changedVpnId) {
2776
2777         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : called for dpId {}, routerName {} "
2778                 + "changed VPN ID {}", dpId, routerName, changedVpnId);
2779         List<MatchInfo> matches = new ArrayList<>();
2780         matches.add(MatchEthernetType.IPV4);
2781         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2782
2783         List<InstructionInfo> instructions = new ArrayList<>();
2784         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2785
2786         String flowRef = getFlowRefSnat(dpId, NwConstants.PSNAT_TABLE, routerName);
2787         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
2788             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2789             NwConstants.COOKIE_SNAT_TABLE, matches, instructions);
2790
2791         LOG.debug("buildSnatFlowEntityWithUpdatedVpnIdForPrimrySwtch : Returning SNAT Flow Entity {}", flowEntity);
2792         return flowEntity;
2793     }
2794
2795     // TODO : Replace this with ITM Rpc once its available with full functionality
2796     protected void installTerminatingServiceTblEntryWithUpdatedVpnId(BigInteger dpnId, String routerName,
2797         long routerId, long changedVpnId, TypedWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
2798
2799         LOG.debug("installTerminatingServiceTblEntryWithUpdatedVpnId : called for switch {}, "
2800             + "routerName {}, BGP VPN ID {}", dpnId, routerName, changedVpnId);
2801         FlowEntity flowEntity = buildTsFlowEntityWithUpdatedVpnId(dpnId, routerName, routerId, changedVpnId,
2802                 extNwProvType);
2803         mdsalManager.addFlow(confTx, flowEntity);
2804     }
2805
2806     private FlowEntity buildTsFlowEntityWithUpdatedVpnId(BigInteger dpId, String routerName,
2807             long routerIdLongVal, long changedVpnId, ProviderTypes extNwProvType) {
2808         LOG.debug("buildTsFlowEntityWithUpdatedVpnId : called for switch {}, routerName {}, BGP VPN ID {}",
2809             dpId, routerName, changedVpnId);
2810         List<MatchInfo> matches = new ArrayList<>();
2811         matches.add(MatchEthernetType.IPV4);
2812
2813         BigInteger tunnelId = BigInteger.valueOf(changedVpnId);
2814         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
2815             tunnelId = natOverVxlanUtil.getRouterVni(routerName, changedVpnId);
2816         }
2817         matches.add(new MatchTunnelId(tunnelId));
2818
2819         List<InstructionInfo> instructions = new ArrayList<>();
2820         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId),
2821             MetaDataUtil.METADATA_MASK_VRFID));
2822         instructions.add(new InstructionGotoTable(NwConstants.OUTBOUND_NAPT_TABLE));
2823         BigInteger routerId = BigInteger.valueOf(routerIdLongVal);
2824         String flowRef = getFlowRefTs(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, routerId.longValue());
2825         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, flowRef,
2826             NatConstants.DEFAULT_TS_FLOW_PRIORITY, flowRef, 0, 0,
2827             NwConstants.COOKIE_TS_TABLE, matches, instructions);
2828         return flowEntity;
2829     }
2830
2831     public void createOutboundTblEntryWithBgpVpn(BigInteger dpnId, long routerId, long changedVpnId,
2832                                                  TypedWriteTransaction<Configuration> writeFlowInvTx) {
2833         LOG.debug("createOutboundTblEntryWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2834             dpnId, routerId, changedVpnId);
2835         FlowEntity tcpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2836                 NwConstants.IP_PROT_TCP);
2837         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing tcp flow {}", tcpFlowEntity);
2838         mdsalManager.addFlow(writeFlowInvTx, tcpFlowEntity);
2839
2840         FlowEntity udpFlowEntity = buildOutboundFlowEntityWithBgpVpn(dpnId, routerId, changedVpnId,
2841                 NwConstants.IP_PROT_UDP);
2842         LOG.debug("createOutboundTblEntryWithBgpVpn : Installing udp flow {}", udpFlowEntity);
2843         mdsalManager.addFlow(writeFlowInvTx, udpFlowEntity);
2844
2845         FlowEntity icmpDropFlow = buildIcmpDropFlow(dpnId, routerId, changedVpnId);
2846         LOG.debug("createOutboundTblEntry: Installing icmp drop flow {}", icmpDropFlow);
2847         mdsalManager.addFlow(writeFlowInvTx, icmpDropFlow);
2848     }
2849
2850     protected FlowEntity buildOutboundFlowEntityWithBgpVpn(BigInteger dpId, long routerId,
2851                                                            long changedVpnId, int protocol) {
2852         LOG.debug("buildOutboundFlowEntityWithBgpVpn : called for dpId {} and routerId {}, BGP VPN ID {}",
2853             dpId, routerId, changedVpnId);
2854         BigInteger cookie = getCookieOutboundFlow(routerId);
2855         List<MatchInfo> matches = new ArrayList<>();
2856         matches.add(MatchEthernetType.IPV4);
2857         matches.add(new MatchIpProtocol((short)protocol));
2858         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2859
2860         List<InstructionInfo> instructions = new ArrayList<>();
2861         List<ActionInfo> actionsInfos = new ArrayList<>();
2862         actionsInfos.add(new ActionPuntToController());
2863         if (snatPuntTimeout != 0) {
2864             actionsInfos.add(getLearnActionForPunt(protocol, snatPuntTimeout, cookie));
2865         }
2866         instructions.add(new InstructionApplyActions(actionsInfos));
2867
2868         String flowRef = getFlowRefOutbound(dpId, NwConstants.OUTBOUND_NAPT_TABLE, routerId, protocol);
2869         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.OUTBOUND_NAPT_TABLE,
2870             flowRef, 5, flowRef, 0, 0, cookie, matches, instructions);
2871         LOG.debug("createOutboundTblEntryWithBgpVpn : returning flowEntity {}", flowEntity);
2872         return flowEntity;
2873     }
2874
2875     public void installNaptPfibEntryWithBgpVpn(BigInteger dpnId, long segmentId, long changedVpnId,
2876                                                TypedWriteTransaction<Configuration> writeFlowInvTx) {
2877         LOG.debug("installNaptPfibEntryWithBgpVpn : called for dpnId {} and segmentId {} ,BGP VPN ID {}",
2878             dpnId, segmentId, changedVpnId);
2879         FlowEntity naptPfibFlowEntity = buildNaptPfibFlowEntityWithUpdatedVpnId(dpnId, segmentId, changedVpnId);
2880         mdsalManager.addFlow(writeFlowInvTx, naptPfibFlowEntity);
2881     }
2882
2883     public FlowEntity buildNaptPfibFlowEntityWithUpdatedVpnId(BigInteger dpId, long segmentId, long changedVpnId) {
2884
2885         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : called for dpId {}, "
2886             + "segmentId {}, BGP VPN ID {}", dpId, segmentId, changedVpnId);
2887         List<MatchInfo> matches = new ArrayList<>();
2888         matches.add(MatchEthernetType.IPV4);
2889         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(changedVpnId), MetaDataUtil.METADATA_MASK_VRFID));
2890
2891         ArrayList<ActionInfo> listActionInfo = new ArrayList<>();
2892         ArrayList<InstructionInfo> instructionInfo = new ArrayList<>();
2893         listActionInfo.add(new ActionNxLoadInPort(BigInteger.ZERO));
2894         listActionInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
2895         instructionInfo.add(new InstructionApplyActions(listActionInfo));
2896
2897         String flowRef = getFlowRefTs(dpId, NwConstants.NAPT_PFIB_TABLE, segmentId);
2898         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.NAPT_PFIB_TABLE, flowRef,
2899             NatConstants.DEFAULT_PSNAT_FLOW_PRIORITY, flowRef, 0, 0,
2900             NwConstants.COOKIE_SNAT_TABLE, matches, instructionInfo);
2901         LOG.debug("buildNaptPfibFlowEntityWithUpdatedVpnId : Returning NaptPFib Flow Entity {}", flowEntity);
2902         return flowEntity;
2903     }
2904
2905     @Override
2906     protected ExternalRoutersListener getDataTreeChangeListener() {
2907         return ExternalRoutersListener.this;
2908     }
2909
2910     protected void installNaptPfibEntriesForExternalSubnets(String routerName, BigInteger dpnId,
2911                                                         @Nullable TypedWriteTransaction<Configuration> writeFlowInvTx) {
2912         Collection<Uuid> externalSubnetIdsForRouter = NatUtil.getExternalSubnetIdsForRouter(dataBroker,
2913                 routerName);
2914         for (Uuid externalSubnetId : externalSubnetIdsForRouter) {
2915             long subnetVpnId = NatUtil.getVpnId(dataBroker, externalSubnetId.getValue());
2916             if (subnetVpnId != -1) {
2917                 LOG.debug("installNaptPfibEntriesForExternalSubnets : called for dpnId {} "
2918                     + "and vpnId {}", dpnId, subnetVpnId);
2919                 installNaptPfibEntry(dpnId, subnetVpnId, writeFlowInvTx);
2920             }
2921         }
2922     }
2923 }