natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/RouterDpnChangeListener.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.base.Optional;
  13 import java.math.BigInteger;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.concurrent.ExecutionException;
  19 import javax.annotation.PostConstruct;
  20 import javax.inject.Inject;
  21 import javax.inject.Singleton;
  22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  23 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  24 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  25 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  26 import org.opendaylight.genius.infra.Datastore.Configuration;
  27 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  28 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  29 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  30 import org.opendaylight.genius.mdsalutil.BucketInfo;
  31 import org.opendaylight.genius.mdsalutil.FlowEntity;
  32 import org.opendaylight.genius.mdsalutil.GroupEntity;
  33 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  34 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  35 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  36 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  37 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  38 import org.opendaylight.netvirt.natservice.api.SnatServiceManager;
  39 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig.NatMode;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  50 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  51 import org.slf4j.Logger;
  52 import org.slf4j.LoggerFactory;
  53
  54 @Singleton
  55 public class RouterDpnChangeListener
  56         extends AsyncDataTreeChangeListenerBase<DpnVpninterfacesList, RouterDpnChangeListener> {
  57
  58     private static final Logger LOG = LoggerFactory.getLogger(RouterDpnChangeListener.class);
  59     private final DataBroker dataBroker;
  60     private final ManagedNewTransactionRunner txRunner;
  61     private final IMdsalApiManager mdsalManager;
  62     private final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer;
  63     private final NaptSwitchHA naptSwitchHA;
  64     private final IdManagerService idManager;
  65     private final INeutronVpnManager nvpnManager;
  66     private final ExternalNetworkGroupInstaller extNetGroupInstaller;
  67     private final IElanService elanManager;
  68     private final JobCoordinator coordinator;
  69     private final SnatServiceManager natServiceManager;
  70     private final NatMode natMode;
  71
  72     @Inject
  73     public RouterDpnChangeListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  74                                    final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
  75                                    final NaptSwitchHA naptSwitchHA,
  76                                    final IdManagerService idManager,
  77                                    final ExternalNetworkGroupInstaller extNetGroupInstaller,
  78                                    final INeutronVpnManager nvpnManager,
  79                                    final SnatServiceManager natServiceManager,
  80                                    final NatserviceConfig config,
  81                                    final IElanService elanManager,
  82                                    final JobCoordinator coordinator) {
  83         super(DpnVpninterfacesList.class, RouterDpnChangeListener.class);
  84         this.dataBroker = dataBroker;
  85         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  86         this.mdsalManager = mdsalManager;
  87         this.snatDefaultRouteProgrammer = snatDefaultRouteProgrammer;
  88         this.naptSwitchHA = naptSwitchHA;
  89         this.idManager = idManager;
  90         this.extNetGroupInstaller = extNetGroupInstaller;
  91         this.nvpnManager = nvpnManager;
  92         this.elanManager = elanManager;
  93         this.natServiceManager = natServiceManager;
  94         this.coordinator = coordinator;
  95         this.natMode = config != null ? config.getNatMode() : NatMode.Controller;
  96     }
  97
  98     @Override
  99     @PostConstruct
 100     public void init() {
 101         LOG.info("{} init", getClass().getSimpleName());
 102         registerListener(LogicalDatastoreType.OPERATIONAL, dataBroker);
 103     }
 104
 105     @Override
 106     protected RouterDpnChangeListener getDataTreeChangeListener() {
 107         return RouterDpnChangeListener.this;
 108     }
 109
 110     @Override
 111     protected InstanceIdentifier<DpnVpninterfacesList> getWildCardPath() {
 112         return InstanceIdentifier.create(NeutronRouterDpns.class).child(RouterDpnList.class)
 113             .child(DpnVpninterfacesList.class);
 114     }
 115
 116     @Override
 117     protected void add(final InstanceIdentifier<DpnVpninterfacesList> identifier, final DpnVpninterfacesList dpnInfo) {
 118         LOG.trace("add : key: {}, value: {}", dpnInfo.key(), dpnInfo);
 119         final String routerUuid = identifier.firstKeyOf(RouterDpnList.class).getRouterId();
 120         BigInteger dpnId = dpnInfo.getDpnId();
 121         //check router is associated to external network
 122         InstanceIdentifier<Routers> id = NatUtil.buildRouterIdentifier(routerUuid);
 123         Optional<Routers> routerData =
 124                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 125                         LogicalDatastoreType.CONFIGURATION, id);
 126         if (routerData.isPresent()) {
 127             Routers router = routerData.get();
 128             Uuid networkId = router.getNetworkId();
 129             if (networkId != null) {
 130                 if (natMode == NatMode.Conntrack) {
 131                     BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterName(dataBroker, router.getRouterName());
 132                     if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 133                         LOG.warn("add : NAPT switch is not selected.");
 134                         return;
 135                     }
 136                     //If it is for NAPT switch skip as the flows would be already programmed.
 137                     if (naptSwitch.equals(dpnId)) {
 138                         LOG.debug("Skipping the notification recived for NAPT switch {}", routerUuid);
 139                         return;
 140                     }
 141                     ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 142                         confTx -> natServiceManager.notify(confTx, router, naptSwitch, dpnId,
 143                             SnatServiceManager.Action.SNAT_ROUTER_ENBL)), LOG, "Error notifying NAT service manager");
 144                 } else {
 145                     Long routerId = NatUtil.getVpnId(dataBroker, routerUuid);
 146                     if (routerId == NatConstants.INVALID_ID) {
 147                         LOG.error("add : Invalid routerId returned for routerName {}", routerUuid);
 148                         return;
 149                     }
 150                     ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker,
 151                             routerUuid, networkId);
 152                     if (extNwProvType == ProviderTypes.FLAT || extNwProvType == ProviderTypes.VLAN) {
 153                         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + networkId, () -> {
 154                             extNetGroupInstaller.installExtNetGroupEntries(networkId, dpnId);
 155                             installDefaultNatRouteForRouterExternalSubnets(dpnId,
 156                                     NatUtil.getExternalSubnetIdsFromExternalIps(router.getExternalIps()));
 157                             return Collections.emptyList();
 158                         });
 159                     }
 160                     coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + router.getRouterName(), () -> {
 161                         LOG.debug("add : Router {} is associated with ext nw {}", routerUuid, networkId);
 162                         Uuid vpnName = NatUtil.getVpnForRouter(dataBroker, routerUuid);
 163                         return Collections.singletonList(
 164                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 165                                 Long vpnId;
 166                                 if (vpnName == null) {
 167                                     LOG.debug("add : Internal vpn associated to router {}", routerUuid);
 168                                     vpnId = routerId;
 169                                     if (vpnId == NatConstants.INVALID_ID) {
 170                                         LOG.error("add : Invalid vpnId returned for routerName {}", routerUuid);
 171                                         return;
 172                                     }
 173                                     LOG.debug("add : Retrieved vpnId {} for router {}", vpnId, routerUuid);
 174                                     //Install default entry in FIB to SNAT table
 175                                     LOG.info(
 176                                         "add : Installing default route in FIB on dpn {} for router {} with vpn {}",
 177                                         dpnId, routerUuid, vpnId);
 178                                     snatDefaultRouteProgrammer.installDefNATRouteInDPN(dpnId, vpnId, confTx);
 179                                 } else {
 180                                     LOG.debug("add : External BGP vpn associated to router {}", routerUuid);
 181                                     vpnId = NatUtil.getVpnId(dataBroker, vpnName.getValue());
 182                                     if (vpnId == NatConstants.INVALID_ID) {
 183                                         LOG.error("add : Invalid vpnId returned for routerName {}", routerUuid);
 184                                         return;
 185                                     }
 186                                     LOG.debug("add : Retrieved vpnId {} for router {}", vpnId, routerUuid);
 187                                     //Install default entry in FIB to SNAT table
 188                                     LOG.debug("add : Installing default route in FIB on dpn {} for routerId {} with "
 189                                         + "vpnId {}...", dpnId, routerUuid, vpnId);
 190                                     snatDefaultRouteProgrammer.installDefNATRouteInDPN(dpnId, vpnId, routerId, confTx);
 191                                 }
 192                                 if (router.isEnableSnat()) {
 193                                     LOG.info("add : SNAT enabled for router {}", routerUuid);
 194                                     if (extNwProvType == null) {
 195                                         LOG.error("add : External Network Provider Type missing");
 196                                         return;
 197                                     }
 198                                     handleSNATForDPN(dpnId, routerUuid, routerId, vpnId, confTx, extNwProvType);
 199                                 } else {
 200                                     LOG.info("add : SNAT is not enabled for router {} to handle addDPN event {}",
 201                                         routerUuid, dpnId);
 202                                 }
 203                             }));
 204                     }, NatConstants.NAT_DJC_MAX_RETRIES);
 205                 } // end of controller based SNAT
 206             }
 207         } else {
 208             LOG.debug("add : Router {} is not associated with External network", routerUuid);
 209         }
 210     }
 211
 212     @Override
 213     protected void remove(InstanceIdentifier<DpnVpninterfacesList> identifier, DpnVpninterfacesList dpnInfo) {
 214         LOG.trace("remove : key: {}, value: {}", dpnInfo.key(), dpnInfo);
 215         final String routerUuid = identifier.firstKeyOf(RouterDpnList.class).getRouterId();
 216         Long routerId = NatUtil.getVpnId(dataBroker, routerUuid);
 217         if (routerId == NatConstants.INVALID_ID) {
 218             LOG.error("REMOVE: Invalid routId returned for routerName {}",routerUuid);
 219             return;
 220         }
 221         BigInteger dpnId = dpnInfo.getDpnId();
 222         //check router is associated to external network
 223         InstanceIdentifier<Routers> id = NatUtil.buildRouterIdentifier(routerUuid);
 224         Optional<Routers> routerData =
 225                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 226                         LogicalDatastoreType.CONFIGURATION, id);
 227         if (routerData.isPresent()) {
 228             Routers router = routerData.get();
 229             Uuid networkId = router.getNetworkId();
 230             if (networkId != null) {
 231                 if (natMode == NatMode.Conntrack) {
 232                     BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterName(dataBroker, router.getRouterName());
 233                     if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 234                         LOG.warn("remove : NAPT switch is not selected.");
 235                         return;
 236                     }
 237                     //If it is for NAPT switch skip as the flows would be already programmed.
 238                     if (naptSwitch.equals(dpnId)) {
 239                         LOG.debug("Skipping the notification recived for NAPT switch {}", routerUuid);
 240                         return;
 241                     }
 242                     ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 243                         confTx -> natServiceManager.notify(confTx, router, naptSwitch, dpnId,
 244                             SnatServiceManager.Action.SNAT_ROUTER_DISBL)), LOG, "Error notifying NAT service manager");
 245                 } else {
 246                     coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + routerUuid, () -> {
 247                         LOG.debug("remove : Router {} is associated with ext nw {}", routerUuid, networkId);
 248                         Uuid vpnName = NatUtil.getVpnForRouter(dataBroker, routerUuid);
 249                         return Collections.singletonList(
 250                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 251                                 Long vpnId;
 252                                 if (vpnName == null) {
 253                                     LOG.debug("remove : Internal vpn associated to router {}", routerUuid);
 254                                     vpnId = routerId;
 255                                     if (vpnId == NatConstants.INVALID_ID) {
 256                                         LOG.error("remove : Invalid vpnId returned for routerName {}", routerUuid);
 257                                         return;
 258                                     }
 259                                     LOG.debug("remove : Retrieved vpnId {} for router {}", vpnId, routerUuid);
 260                                     //Remove default entry in FIB
 261                                     LOG.debug("remove : Removing default route in FIB on dpn {} for vpn {} ...", dpnId,
 262                                         vpnName);
 263                                     snatDefaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, vpnId, confTx);
 264                                 } else {
 265                                     LOG.debug("remove : External vpn associated to router {}", routerUuid);
 266                                     vpnId = NatUtil.getVpnId(dataBroker, vpnName.getValue());
 267                                     if (vpnId == NatConstants.INVALID_ID) {
 268                                         LOG.error("remove : Invalid vpnId returned for routerName {}", routerUuid);
 269                                         return;
 270                                     }
 271                                     LOG.debug("remove : Retrieved vpnId {} for router {}", vpnId, routerUuid);
 272                                     //Remove default entry in FIB
 273                                     LOG.debug("remove : Removing default route in FIB on dpn {} for vpn {} ...", dpnId,
 274                                         vpnName);
 275                                     snatDefaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, vpnId, routerId, confTx);
 276                                 }
 277
 278                                 if (router.isEnableSnat()) {
 279                                     LOG.info("remove : SNAT enabled for router {}", routerUuid);
 280                                     removeSNATFromDPN(dpnId, routerUuid, routerId, vpnId, networkId, confTx);
 281                                 } else {
 282                                     LOG.info("remove : SNAT is not enabled for router {} to handle removeDPN event {}",
 283                                         routerUuid, dpnId);
 284                                 }
 285                             }));
 286                     }, NatConstants.NAT_DJC_MAX_RETRIES);
 287                 } // end of controller based SNAT
 288             }
 289         }
 290     }
 291
 292     @Override
 293     protected void update(InstanceIdentifier<DpnVpninterfacesList> identifier, DpnVpninterfacesList original,
 294                           DpnVpninterfacesList update) {
 295         LOG.trace("Update key: {}, original: {}, update: {}", update.key(), original, update);
 296     }
 297
 298     void handleSNATForDPN(BigInteger dpnId, String routerName, long routerId, Long routerVpnId,
 299         TypedReadWriteTransaction<Configuration> confTx, ProviderTypes extNwProvType) {
 300        //Check if primary and secondary switch are selected, If not select the role
 301         //Install select group to NAPT switch
 302         //Install default miss entry to NAPT switch
 303         BigInteger naptSwitch;
 304         try {
 305             BigInteger naptId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 306             if (naptId == null || naptId.equals(BigInteger.ZERO) || !naptSwitchHA.getSwitchStatus(naptId)) {
 307                 LOG.debug("handleSNATForDPN : No NaptSwitch is selected for router {}", routerName);
 308                 naptSwitch = dpnId;
 309                 boolean naptstatus = naptSwitchHA.updateNaptSwitch(routerName, naptSwitch);
 310                 if (!naptstatus) {
 311                     LOG.error("handleSNATForDPN : Failed to update newNaptSwitch {} for routername {}",
 312                             naptSwitch, routerName);
 313                     return;
 314                 }
 315                 LOG.debug("handleSNATForDPN : Switch {} is elected as NaptSwitch for router {}", dpnId, routerName);
 316
 317                 // When NAPT switch is elected during first VM comes up for the given Router
 318                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanManager, extNwProvType)) {
 319                     NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager,
 320                             idManager, NatConstants.ODL_VNI_POOL_NAME);
 321                 }
 322
 323                 Routers extRouters = NatUtil.getRoutersFromConfigDS(dataBroker, routerName);
 324                 if (extRouters != null) {
 325                     NatUtil.createRouterIdsConfigDS(dataBroker, routerId, routerName);
 326                     naptSwitchHA.subnetRegisterMapping(extRouters, routerId);
 327                 }
 328
 329                 naptSwitchHA.installSnatFlows(routerName, routerId, naptSwitch, routerVpnId, confTx);
 330
 331                 // Install miss entry (table 26) pointing to table 46
 332                 FlowEntity flowEntity = naptSwitchHA.buildSnatFlowEntityForNaptSwitch(dpnId, routerName,
 333                         routerVpnId, NatConstants.ADD_FLOW);
 334                 if (flowEntity == null) {
 335                     LOG.error("handleSNATForDPN : Failed to populate flowentity for router {} with dpnId {}",
 336                             routerName, dpnId);
 337                     return;
 338                 }
 339                 LOG.debug("handleSNATForDPN : Successfully installed flow for dpnId {} router {}", dpnId, routerName);
 340                 mdsalManager.addFlow(confTx, flowEntity);
 341                 //Removing primary flows from old napt switch
 342                 if (naptId != null && !naptId.equals(BigInteger.ZERO)) {
 343                     LOG.debug("handleSNATForDPN : Removing primary flows from old napt switch {} for router {}",
 344                             naptId, routerName);
 345                     naptSwitchHA.removeSnatFlowsInOldNaptSwitch(routerName, routerId, naptId, null, confTx);
 346                 }
 347             } else if (naptId.equals(dpnId)) {
 348                 LOG.debug("handleSNATForDPN : NaptSwitch {} gone down during cluster reboot came alive", naptId);
 349             } else {
 350                 naptSwitch = naptId;
 351                 LOG.debug("handleSNATForDPN : Napt switch with Id {} is already elected for router {}",
 352                         naptId, routerName);
 353
 354                 //installing group
 355                 List<BucketInfo> bucketInfo = naptSwitchHA.handleGroupInNeighborSwitches(dpnId,
 356                         routerName, routerId, naptSwitch);
 357                 naptSwitchHA.installSnatGroupEntry(dpnId, bucketInfo, routerName);
 358
 359                 // Install miss entry (table 26) pointing to group
 360                 long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 361                 FlowEntity flowEntity =
 362                         naptSwitchHA.buildSnatFlowEntity(dpnId, routerName, groupId,
 363                                 routerVpnId, NatConstants.ADD_FLOW);
 364                 if (flowEntity == null) {
 365                     LOG.error("handleSNATForDPN : Failed to populate flowentity for router {} with dpnId {} groupId {}",
 366                             routerName, dpnId, groupId);
 367                     return;
 368                 }
 369                 LOG.debug("handleSNATForDPN : Successfully installed flow for dpnId {} router {} group {}",
 370                         dpnId, routerName, groupId);
 371                 mdsalManager.addFlow(confTx, flowEntity);
 372             }
 373
 374         } catch (InterruptedException | ExecutionException e) {
 375             LOG.error("handleSNATForDPN : Exception in handleSNATForDPN", e);
 376         }
 377     }
 378
 379     // TODO Clean up the exception handling
 380     @SuppressWarnings("checkstyle:IllegalCatch")
 381     void removeSNATFromDPN(BigInteger dpnId, String routerName, long routerId, long routerVpnId,
 382             Uuid extNetworkId, TypedReadWriteTransaction<Configuration> confTx) {
 383         //irrespective of naptswitch or non-naptswitch, SNAT default miss entry need to be removed
 384         //remove miss entry to NAPT switch
 385         //if naptswitch elect new switch and install Snat flows and remove those flows in oldnaptswitch
 386
 387         Collection<String> externalIpCache = NatUtil.getExternalIpsForRouter(dataBroker, routerId);
 388         ProviderTypes extNwProvType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNetworkId);
 389         if (extNwProvType == null) {
 390             return;
 391         }
 392         //Get the external IP labels other than VXLAN provider type. Since label is not applicable for VXLAN
 393         Map<String, Long> externalIpLabel;
 394         if (extNwProvType == ProviderTypes.VXLAN) {
 395             externalIpLabel = null;
 396         } else {
 397             externalIpLabel = NatUtil.getExternalIpsLabelForRouter(dataBroker, routerId);
 398         }
 399         BigInteger naptSwitch = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 400         if (naptSwitch == null || naptSwitch.equals(BigInteger.ZERO)) {
 401             LOG.error("removeSNATFromDPN : No naptSwitch is selected for router {}", routerName);
 402             return;
 403         }
 404         try {
 405             boolean naptStatus =
 406                 naptSwitchHA.isNaptSwitchDown(routerName, routerId, dpnId, naptSwitch, routerVpnId,
 407                         externalIpCache, confTx);
 408             if (!naptStatus) {
 409                 LOG.debug("removeSNATFromDPN: Switch with DpnId {} is not naptSwitch for router {}",
 410                     dpnId, routerName);
 411                 long groupId = NatUtil.createGroupId(NatUtil.getGroupIdKey(routerName), idManager);
 412                 FlowEntity flowEntity = null;
 413                 try {
 414                     flowEntity = naptSwitchHA.buildSnatFlowEntity(dpnId, routerName, groupId, routerVpnId,
 415                         NatConstants.DEL_FLOW);
 416                     if (flowEntity == null) {
 417                         LOG.error("removeSNATFromDPN : Failed to populate flowentity for router:{} "
 418                                 + "with dpnId:{} groupId:{}", routerName, dpnId, groupId);
 419                         return;
 420                     }
 421                     LOG.debug("removeSNATFromDPN : Removing default SNAT miss entry flow entity {}", flowEntity);
 422                     mdsalManager.removeFlow(confTx, flowEntity);
 423
 424                 } catch (Exception ex) {
 425                     LOG.error("removeSNATFromDPN : Failed to remove default SNAT miss entry flow entity {}",
 426                         flowEntity, ex);
 427                     return;
 428                 }
 429                 LOG.debug("removeSNATFromDPN : Removed default SNAT miss entry flow for dpnID {} with routername {}",
 430                     dpnId, routerName);
 431
 432                 //remove group
 433                 GroupEntity groupEntity = null;
 434                 try {
 435                     groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, routerName,
 436                         GroupTypes.GroupAll, Collections.emptyList() /*listBucketInfo*/);
 437                     LOG.info("removeSNATFromDPN : Removing NAPT GroupEntity:{}", groupEntity);
 438                     mdsalManager.removeGroup(groupEntity);
 439                 } catch (Exception ex) {
 440                     LOG.error("removeSNATFromDPN : Failed to remove group entity {}", groupEntity, ex);
 441                     return;
 442                 }
 443                 LOG.debug("removeSNATFromDPN : Removed default SNAT miss entry flow for dpnID {} with routerName {}",
 444                     dpnId, routerName);
 445             } else {
 446                 naptSwitchHA.removeSnatFlowsInOldNaptSwitch(routerName, routerId, naptSwitch,
 447                         externalIpLabel, confTx);
 448                 //remove table 26 flow ppointing to table46
 449                 FlowEntity flowEntity = null;
 450                 try {
 451                     flowEntity = naptSwitchHA.buildSnatFlowEntityForNaptSwitch(dpnId, routerName, routerVpnId,
 452                         NatConstants.DEL_FLOW);
 453                     if (flowEntity == null) {
 454                         LOG.error("removeSNATFromDPN : Failed to populate flowentity for router {} with dpnId {}",
 455                                 routerName, dpnId);
 456                         return;
 457                     }
 458                     LOG.debug("removeSNATFromDPN : Removing default SNAT miss entry flow entity for router {} with "
 459                         + "dpnId {} in napt switch {}", routerName, dpnId, naptSwitch);
 460                     mdsalManager.removeFlow(confTx, flowEntity);
 461
 462                 } catch (Exception ex) {
 463                     LOG.error("removeSNATFromDPN : Failed to remove default SNAT miss entry flow entity {}",
 464                         flowEntity, ex);
 465                     return;
 466                 }
 467                 LOG.debug("removeSNATFromDPN : Removed default SNAT miss entry flow for dpnID {} with routername {}",
 468                     dpnId, routerName);
 469
 470                 //best effort to check IntExt model
 471                 naptSwitchHA.bestEffortDeletion(routerId, routerName, externalIpLabel, confTx);
 472             }
 473         } catch (InterruptedException | ExecutionException e) {
 474             LOG.error("removeSNATFromDPN : Exception while handling naptSwitch down for router {}", routerName, e);
 475         }
 476     }
 477
 478     private void installDefaultNatRouteForRouterExternalSubnets(BigInteger dpnId, Collection<Uuid> externalSubnetIds) {
 479         if (externalSubnetIds == null) {
 480             LOG.error("installDefaultNatRouteForRouterExternalSubnets : No external subnets for router");
 481             return;
 482         }
 483
 484         for (Uuid subnetId : externalSubnetIds) {
 485             long vpnIdForSubnet = NatUtil.getExternalSubnetVpnId(dataBroker, subnetId);
 486             if (vpnIdForSubnet != NatConstants.INVALID_ID) {
 487                 LOG.info("installDefaultNatRouteForRouterExternalSubnets : Installing default routes in FIB on dpn {} "
 488                         + "for subnetId {} with vpnId {}", dpnId, subnetId, vpnIdForSubnet);
 489                 snatDefaultRouteProgrammer.installDefNATRouteInDPN(dpnId, vpnIdForSubnet, subnetId.getValue());
 490             } else {
 491                 LOG.debug("installDefaultNatRouteForRouterExternalSubnets : No vpnID for subnet {} found", subnetId);
 492             }
 493         }
 494     }
 495 }