natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/FloatingIPListener.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import com.google.common.base.Optional;
  11 import java.math.BigInteger;
  12 import java.net.InetAddress;
  13 import java.net.UnknownHostException;
  14 import java.util.ArrayList;
  15 import java.util.Collections;
  16 import java.util.List;
  17 import javax.annotation.PostConstruct;
  18 import javax.inject.Inject;
  19 import javax.inject.Singleton;
  20 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  21 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  22 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  23 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  24 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  25 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  26 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  27 import org.opendaylight.genius.mdsalutil.ActionInfo;
  28 import org.opendaylight.genius.mdsalutil.FlowEntity;
  29 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfo;
  32 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  33 import org.opendaylight.genius.mdsalutil.NwConstants;
  34 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  35 import org.opendaylight.genius.mdsalutil.actions.ActionSetDestinationIp;
  36 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  37 import org.opendaylight.genius.mdsalutil.actions.ActionSetSourceIp;
  38 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  39 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  40 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  41 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  42 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetDestination;
  43 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Source;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  47 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  50 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.FloatingIpInfo;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.RouterPorts;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.Ports;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.PortsBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.PortsKey;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMapBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMapKey;
  64 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  65 import org.slf4j.Logger;
  66 import org.slf4j.LoggerFactory;
  67
  68 @Singleton
  69 public class FloatingIPListener extends AsyncDataTreeChangeListenerBase<InternalToExternalPortMap, FloatingIPListener> {
  70     private static final Logger LOG = LoggerFactory.getLogger(FloatingIPListener.class);
  71     private final DataBroker dataBroker;
  72     private final ManagedNewTransactionRunner txRunner;
  73     private final IMdsalApiManager mdsalManager;
  74     private final OdlInterfaceRpcService interfaceManager;
  75     private final FloatingIPHandler floatingIPHandler;
  76     private final SNATDefaultRouteProgrammer defaultRouteProgrammer;
  77     private final JobCoordinator coordinator;
  78
  79     @Inject
  80     public FloatingIPListener(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  81                               final OdlInterfaceRpcService interfaceManager,
  82                               final FloatingIPHandler floatingIPHandler,
  83                               final SNATDefaultRouteProgrammer snatDefaultRouteProgrammer,
  84                               final JobCoordinator coordinator) {
  85         super(InternalToExternalPortMap.class, FloatingIPListener.class);
  86         this.dataBroker = dataBroker;
  87         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  88         this.mdsalManager = mdsalManager;
  89         this.interfaceManager = interfaceManager;
  90         this.floatingIPHandler = floatingIPHandler;
  91         this.defaultRouteProgrammer = snatDefaultRouteProgrammer;
  92         this.coordinator = coordinator;
  93     }
  94
  95     @Override
  96     @PostConstruct
  97     public void init() {
  98         LOG.info("{} init", getClass().getSimpleName());
  99         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 100     }
 101
 102     @Override
 103     protected InstanceIdentifier<InternalToExternalPortMap> getWildCardPath() {
 104         return InstanceIdentifier.create(FloatingIpInfo.class).child(RouterPorts.class).child(Ports.class)
 105                 .child(InternalToExternalPortMap.class);
 106     }
 107
 108     @Override
 109     protected FloatingIPListener getDataTreeChangeListener() {
 110         return FloatingIPListener.this;
 111     }
 112
 113     @Override
 114     protected void add(final InstanceIdentifier<InternalToExternalPortMap> identifier,
 115                        final InternalToExternalPortMap mapping) {
 116         LOG.trace("FloatingIPListener add ip mapping method - key: {} value: {}",mapping.getKey(), mapping);
 117         processFloatingIPAdd(identifier, mapping);
 118     }
 119
 120     @Override
 121     protected void remove(InstanceIdentifier<InternalToExternalPortMap> identifier, InternalToExternalPortMap mapping) {
 122         LOG.trace("FloatingIPListener remove ip mapping method - kkey: {} value: {}",mapping.getKey(), mapping);
 123         processFloatingIPDel(identifier, mapping);
 124     }
 125
 126     @Override
 127     protected void update(InstanceIdentifier<InternalToExternalPortMap> identifier, InternalToExternalPortMap
 128             original, InternalToExternalPortMap update) {
 129         LOG.trace("FloatingIPListener update ip mapping method - key: {}, original: {}, update: {}",
 130                 update.getKey(), original, update);
 131     }
 132
 133     private FlowEntity buildPreDNATFlowEntity(BigInteger dpId, InternalToExternalPortMap mapping, long routerId, long
 134             associatedVpn) {
 135         String externalIp = mapping.getExternalIp();
 136         Uuid floatingIpId = mapping.getExternalId();
 137         //Get the FIP MAC address for DNAT
 138         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 139         if (floatingIpPortMacAddress == null) {
 140             LOG.error("buildPreDNATFlowEntity : Unable to retrieve floatingIpPortMacAddress from floating IP UUID {} "
 141                     + "for floating IP {}", floatingIpId, externalIp);
 142             return null;
 143         }
 144         LOG.debug("buildPreDNATFlowEntity : Bulding DNAT Flow entity for ip {} ", externalIp);
 145         long segmentId = associatedVpn == NatConstants.INVALID_ID ? routerId : associatedVpn;
 146         LOG.debug("buildPreDNATFlowEntity : Segment id {} in build preDNAT Flow", segmentId);
 147
 148         List<MatchInfo> matches = new ArrayList<>();
 149         matches.add(MatchEthernetType.IPV4);
 150
 151         matches.add(new MatchIpv4Destination(externalIp, "32"));
 152         //Match Destination Floating IP MAC Address on table = 25 (PDNAT_TABLE)
 153         matches.add(new MatchEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 154
 155 //        matches.add(new MatchMetadata(
 156 //                BigInteger.valueOf(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 157         List<ActionInfo> actionsInfos = new ArrayList<>();
 158         String internalIp = mapping.getInternalIp();
 159         actionsInfos.add(new ActionSetDestinationIp(internalIp, "32"));
 160
 161         List<InstructionInfo> instructions = new ArrayList<>();
 162         instructions.add(new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(segmentId),
 163                 MetaDataUtil.METADATA_MASK_VRFID));
 164         instructions.add(new InstructionApplyActions(actionsInfos));
 165         instructions.add(new InstructionGotoTable(NwConstants.DNAT_TABLE));
 166
 167         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.PDNAT_TABLE, routerId, externalIp);
 168
 169         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PDNAT_TABLE, flowRef,
 170                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 171                 NwConstants.COOKIE_DNAT_TABLE, matches, instructions);
 172
 173         return flowEntity;
 174     }
 175
 176     private FlowEntity buildDNATFlowEntity(BigInteger dpId, InternalToExternalPortMap mapping, long routerId, long
 177             associatedVpn) {
 178         String externalIp = mapping.getExternalIp();
 179         LOG.info("buildDNATFlowEntity : Bulding DNAT Flow entity for ip {} ", externalIp);
 180
 181         long segmentId = associatedVpn == NatConstants.INVALID_ID ? routerId : associatedVpn;
 182         LOG.debug("buildDNATFlowEntity : Segment id {} in build DNAT", segmentId);
 183
 184         List<MatchInfo> matches = new ArrayList<>();
 185         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 186
 187         matches.add(MatchEthernetType.IPV4);
 188         String internalIp = mapping.getInternalIp();
 189         matches.add(new MatchIpv4Destination(internalIp, "32"));
 190
 191         List<ActionInfo> actionsInfos = new ArrayList<>();
 192 //        actionsInfos.add(new ActionSetDestinationIp(internalIp, "32"));
 193
 194         List<InstructionInfo> instructions = new ArrayList<>();
 195 //        instructions.add(new InstructionWriteMetadata(BigInteger.valueOf
 196 //                (routerId), MetaDataUtil.METADATA_MASK_VRFID));
 197         actionsInfos.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 198         instructions.add(new InstructionApplyActions(actionsInfos));
 199         //instructions.add(new InstructionGotoTable(NatConstants.L3_FIB_TABLE));
 200
 201         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.DNAT_TABLE, routerId, internalIp);
 202
 203         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.DNAT_TABLE, flowRef,
 204                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 205                 NwConstants.COOKIE_DNAT_TABLE, matches, instructions);
 206
 207         return flowEntity;
 208
 209     }
 210
 211     private FlowEntity buildPreSNATFlowEntity(BigInteger dpId, String internalIp, String externalIp, long vpnId, long
 212             routerId, long associatedVpn) {
 213
 214         LOG.debug("buildPreSNATFlowEntity : Building PSNAT Flow entity for ip {} ", internalIp);
 215
 216         long segmentId = associatedVpn == NatConstants.INVALID_ID ? routerId : associatedVpn;
 217
 218         LOG.debug("buildPreSNATFlowEntity : Segment id {} in build preSNAT flow", segmentId);
 219
 220         List<MatchInfo> matches = new ArrayList<>();
 221         matches.add(MatchEthernetType.IPV4);
 222
 223         matches.add(new MatchIpv4Source(internalIp, "32"));
 224
 225         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(segmentId), MetaDataUtil.METADATA_MASK_VRFID));
 226
 227         List<ActionInfo> actionsInfos = new ArrayList<>();
 228         actionsInfos.add(new ActionSetSourceIp(externalIp, "32"));
 229
 230         List<InstructionInfo> instructions = new ArrayList<>();
 231         instructions.add(
 232                 new InstructionWriteMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 233         instructions.add(new InstructionApplyActions(actionsInfos));
 234         instructions.add(new InstructionGotoTable(NwConstants.SNAT_TABLE));
 235
 236         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.PSNAT_TABLE, routerId, internalIp);
 237
 238         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 239                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 240                 NwConstants.COOKIE_DNAT_TABLE, matches, instructions);
 241
 242         return flowEntity;
 243     }
 244
 245     private FlowEntity buildSNATFlowEntity(BigInteger dpId, InternalToExternalPortMap mapping, long vpnId, Uuid
 246             externalNetworkId) {
 247         String internalIp = mapping.getInternalIp();
 248         LOG.debug("buildSNATFlowEntity : Building SNAT Flow entity for ip {} ", internalIp);
 249
 250         ProviderTypes provType = NatUtil.getProviderTypefromNetworkId(dataBroker, externalNetworkId);
 251         if (provType == null) {
 252             LOG.error("buildSNATFlowEntity : Unable to get Network Provider Type for network {}", externalNetworkId);
 253             return null;
 254         }
 255
 256         List<MatchInfo> matches = new ArrayList<>();
 257         matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
 258         matches.add(MatchEthernetType.IPV4);
 259         String externalIp = mapping.getExternalIp();
 260         matches.add(new MatchIpv4Source(externalIp, "32"));
 261
 262         List<ActionInfo> actionsInfo = new ArrayList<>();
 263         Uuid floatingIpId = mapping.getExternalId();
 264         String macAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 265         if (macAddress != null) {
 266             actionsInfo.add(new ActionSetFieldEthernetSource(new MacAddress(macAddress)));
 267         } else {
 268             LOG.warn("buildSNATFlowEntity : No MAC address found for floating IP {}", externalIp);
 269         }
 270
 271         LOG.trace("buildSNATFlowEntity : External Network Provider Type is {}, resubmit to FIB", provType.toString());
 272         actionsInfo.add(new ActionNxResubmit(NwConstants.L3_FIB_TABLE));
 273         List<InstructionInfo> instructions = new ArrayList<>();
 274         instructions.add(new InstructionApplyActions(actionsInfo));
 275         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.SNAT_TABLE, vpnId, externalIp);
 276
 277         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.SNAT_TABLE, flowRef,
 278                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 279                 NwConstants.COOKIE_DNAT_TABLE, matches, instructions);
 280
 281         return flowEntity;
 282
 283     }
 284
 285     private void createDNATTblEntry(BigInteger dpnId, InternalToExternalPortMap mapping, long routerId,
 286                                     long associatedVpnId, WriteTransaction writeFlowInvTx) {
 287         FlowEntity preFlowEntity = buildPreDNATFlowEntity(dpnId, mapping, routerId, associatedVpnId);
 288         if (preFlowEntity == null) {
 289             LOG.error("createDNATTblEntry : Flow entity received as NULL. "
 290                     + "Cannot proceed with installation of Pre-DNAT flow table {} --> table {} on DpnId {}",
 291                     NwConstants.PDNAT_TABLE, NwConstants.DNAT_TABLE, dpnId);
 292         } else {
 293             mdsalManager.addFlowToTx(preFlowEntity, writeFlowInvTx);
 294             FlowEntity flowEntity = buildDNATFlowEntity(dpnId, mapping, routerId, associatedVpnId);
 295             if (flowEntity != null) {
 296                 mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 297             }
 298         }
 299     }
 300
 301     private void removeDNATTblEntry(BigInteger dpnId, String internalIp, String externalIp, long routerId,
 302                                     WriteTransaction removeFlowInvTx) {
 303         FlowEntity preFlowEntity = buildPreDNATDeleteFlowEntity(dpnId, externalIp, routerId);
 304         mdsalManager.removeFlowToTx(preFlowEntity, removeFlowInvTx);
 305
 306         FlowEntity flowEntity = buildDNATDeleteFlowEntity(dpnId, internalIp, routerId);
 307         if (flowEntity != null) {
 308             mdsalManager.removeFlowToTx(flowEntity, removeFlowInvTx);
 309         }
 310     }
 311
 312     private void createSNATTblEntry(BigInteger dpnId, InternalToExternalPortMap mapping, long vpnId, long routerId,
 313                                     long associatedVpnId, Uuid externalNetworkId, WriteTransaction writeFlowInvTx) {
 314         FlowEntity preFlowEntity = buildPreSNATFlowEntity(dpnId, mapping.getInternalIp(), mapping.getExternalIp(),
 315             vpnId, routerId, associatedVpnId);
 316         mdsalManager.addFlowToTx(preFlowEntity, writeFlowInvTx);
 317
 318         FlowEntity flowEntity = buildSNATFlowEntity(dpnId, mapping, vpnId, externalNetworkId);
 319         if (flowEntity != null) {
 320             mdsalManager.addFlowToTx(flowEntity, writeFlowInvTx);
 321         }
 322     }
 323
 324     private void removeSNATTblEntry(BigInteger dpnId, String internalIp, String externalIp, long routerId, long vpnId,
 325                                     WriteTransaction removeFlowInvTx) {
 326         FlowEntity preFlowEntity = buildPreSNATDeleteFlowEntity(dpnId, internalIp, routerId);
 327         mdsalManager.removeFlowToTx(preFlowEntity, removeFlowInvTx);
 328
 329         FlowEntity flowEntity = buildSNATDeleteFlowEntity(dpnId, externalIp, vpnId);
 330         if (flowEntity != null) {
 331             mdsalManager.removeFlowToTx(flowEntity, removeFlowInvTx);
 332         }
 333     }
 334
 335     private Uuid getExtNetworkId(final InstanceIdentifier<RouterPorts> portIid,
 336                                  LogicalDatastoreType dataStoreType) {
 337         Optional<RouterPorts> rtrPort =
 338                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 339                         dataStoreType, portIid);
 340         if (!rtrPort.isPresent()) {
 341             LOG.error("getExtNetworkId : Unable to read router port entry for {}", portIid);
 342             return null;
 343         }
 344
 345         Uuid extNwId = rtrPort.get().getExternalNetworkId();
 346         return extNwId;
 347     }
 348
 349     private long getVpnId(Uuid extNwId, Uuid floatingIpExternalId) {
 350         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpExternalId);
 351         if (subnetId != null) {
 352             long vpnId = NatUtil.getVpnId(dataBroker, subnetId.getValue());
 353             if (vpnId != NatConstants.INVALID_ID) {
 354                 LOG.debug("getVpnId : Got vpnId {} for floatingIpExternalId {}", vpnId, floatingIpExternalId);
 355                 return vpnId;
 356             }
 357         }
 358
 359         InstanceIdentifier<Networks> nwId = InstanceIdentifier.builder(ExternalNetworks.class).child(Networks.class,
 360                 new NetworksKey(extNwId)).build();
 361         Optional<Networks> nw =
 362                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 363                         LogicalDatastoreType.CONFIGURATION, nwId);
 364         if (!nw.isPresent()) {
 365             LOG.error("getVpnId : Unable to read external network for {}", extNwId);
 366             return NatConstants.INVALID_ID;
 367         }
 368
 369         Uuid vpnUuid = nw.get().getVpnid();
 370         if (vpnUuid == null) {
 371             LOG.error("getVpnId : Unable to read vpn from External network: {}", extNwId);
 372             return NatConstants.INVALID_ID;
 373         }
 374
 375         //Get the id using the VPN UUID (also vpn instance name)
 376         return NatUtil.getVpnId(dataBroker, vpnUuid.getValue());
 377     }
 378
 379     private void processFloatingIPAdd(final InstanceIdentifier<InternalToExternalPortMap> identifier,
 380                                       final InternalToExternalPortMap mapping) {
 381         LOG.trace("processFloatingIPAdd key: {}, value: {}", mapping.getKey(), mapping);
 382
 383         final String routerId = identifier.firstKeyOf(RouterPorts.class).getRouterId();
 384         final PortsKey pKey = identifier.firstKeyOf(Ports.class);
 385         String interfaceName = pKey.getPortName();
 386
 387         InstanceIdentifier<RouterPorts> portIid = identifier.firstIdentifierOf(RouterPorts.class);
 388         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + mapping.getKey(), () -> Collections.singletonList(
 389                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 390                     tx -> createNATFlowEntries(interfaceName, mapping, portIid, routerId, tx))),
 391                 NatConstants.NAT_DJC_MAX_RETRIES);
 392     }
 393
 394     private void processFloatingIPDel(final InstanceIdentifier<InternalToExternalPortMap> identifier,
 395                                       final InternalToExternalPortMap mapping) {
 396         LOG.trace("processFloatingIPDel : key: {}, value: {}", mapping.getKey(), mapping);
 397
 398         final String routerId = identifier.firstKeyOf(RouterPorts.class).getRouterId();
 399         final PortsKey pKey = identifier.firstKeyOf(Ports.class);
 400         String interfaceName = pKey.getPortName();
 401
 402         InstanceIdentifier<RouterPorts> portIid = identifier.firstIdentifierOf(RouterPorts.class);
 403         coordinator.enqueueJob(NatConstants.NAT_DJC_PREFIX + mapping.getKey(), () -> Collections.singletonList(
 404                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 405                     tx -> removeNATFlowEntries(interfaceName, mapping, portIid, routerId, null, tx))),
 406                 NatConstants.NAT_DJC_MAX_RETRIES);
 407     }
 408
 409     private InetAddress getInetAddress(String ipAddr) {
 410         InetAddress ipAddress = null;
 411         try {
 412             ipAddress = InetAddress.getByName(ipAddr);
 413         } catch (UnknownHostException e) {
 414             LOG.error("getInetAddress : UnknowHostException for ip {}", ipAddr, e);
 415         }
 416         return ipAddress;
 417     }
 418
 419     private boolean validateIpMapping(InternalToExternalPortMap mapping) {
 420         return getInetAddress(mapping.getInternalIp()) != null && getInetAddress(mapping.getExternalIp()) != null;
 421     }
 422
 423     void createNATFlowEntries(String interfaceName, final InternalToExternalPortMap mapping,
 424                               final InstanceIdentifier<RouterPorts> portIid, final String routerName,
 425                               WriteTransaction writeFlowInvTx) {
 426         if (!validateIpMapping(mapping)) {
 427             LOG.error("createNATFlowEntries : Not a valid ip addresses in the mapping {}", mapping);
 428             return;
 429         }
 430
 431         //Get the DPN on which this interface resides
 432         BigInteger dpnId = NatUtil.getDpnForInterface(interfaceManager, interfaceName);
 433
 434         if (dpnId.equals(BigInteger.ZERO)) {
 435             LOG.warn("createNATFlowEntries : No DPN for interface {}. NAT flow entries for ip mapping {} will "
 436                 + "not be installed", interfaceName, mapping);
 437             return;
 438         }
 439
 440         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 441         if (routerId == NatConstants.INVALID_ID) {
 442             LOG.error("createNATFlowEntries : Could not retrieve router id for {} to create NAT Flow entries",
 443                     routerName);
 444             return;
 445         }
 446         //Check if the router to vpn association is present
 447         //long associatedVpnId = NatUtil.getAssociatedVpn(dataBroker, routerName);
 448         Uuid associatedVpn = NatUtil.getVpnForRouter(dataBroker, routerName);
 449         long associatedVpnId = NatConstants.INVALID_ID;
 450         if (associatedVpn == null) {
 451             LOG.debug("createNATFlowEntries : Router {} is not assicated with any BGP VPN instance", routerName);
 452         } else {
 453             LOG.debug("createNATFlowEntries : Router {} is associated with VPN Instance with Id {}",
 454                     routerName, associatedVpn);
 455             associatedVpnId = NatUtil.getVpnId(dataBroker, associatedVpn.getValue());
 456             LOG.debug("createNATFlowEntries : vpninstance Id is {} for VPN {}", associatedVpnId, associatedVpn);
 457             //routerId = associatedVpnId;
 458         }
 459
 460         Uuid extNwId = getExtNetworkId(portIid, LogicalDatastoreType.CONFIGURATION);
 461         if (extNwId == null) {
 462             LOG.error("createNATFlowEntries : External network associated with interface {} could not be retrieved",
 463                 interfaceName);
 464             return;
 465         }
 466         long vpnId = getVpnId(extNwId, mapping.getExternalId());
 467         if (vpnId < 0) {
 468             LOG.error("createNATFlowEntries : No VPN associated with Ext nw {}. Unable to create SNAT table entry "
 469                     + "for fixed ip {}", extNwId, mapping.getInternalIp());
 470             return;
 471         }
 472         //Install the DNAT default FIB flow L3_FIB_TABLE (21) -> PSNAT_TABLE (26) if SNAT is disabled
 473         boolean isSnatEnabled = NatUtil.isSnatEnabledForRouterId(dataBroker, routerName);
 474         if (!isSnatEnabled) {
 475             addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, writeFlowInvTx, true);
 476         }
 477         //Create the DNAT and SNAT table entries
 478         createDNATTblEntry(dpnId, mapping, routerId, associatedVpnId, writeFlowInvTx);
 479         createSNATTblEntry(dpnId, mapping, vpnId, routerId, associatedVpnId, extNwId, writeFlowInvTx);
 480         floatingIPHandler.onAddFloatingIp(dpnId, routerName, routerId, extNwId, interfaceName, mapping, writeFlowInvTx);
 481     }
 482
 483     void createNATFlowEntries(BigInteger dpnId,  String interfaceName, String routerName, Uuid externalNetworkId,
 484                               InternalToExternalPortMap mapping, WriteTransaction writeFlowInvTx) {
 485         String internalIp = mapping.getInternalIp();
 486         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 487         if (routerId == NatConstants.INVALID_ID) {
 488             LOG.error("createNATFlowEntries : Could not retrieve router id for {} to create NAT Flow entries",
 489                     routerName);
 490             return;
 491         }
 492         //Check if the router to vpn association is present
 493         long associatedVpnId = NatUtil.getAssociatedVpn(dataBroker, routerName);
 494         if (associatedVpnId == NatConstants.INVALID_ID) {
 495             LOG.debug("createNATFlowEntries : Router {} is not assicated with any BGP VPN instance", routerName);
 496         } else {
 497             LOG.debug("createNATFlowEntries : Router {} is associated with VPN Instance with Id {}",
 498                 routerName, associatedVpnId);
 499             //routerId = associatedVpnId;
 500         }
 501
 502         long vpnId = getVpnId(externalNetworkId, mapping.getExternalId());
 503         if (vpnId < 0) {
 504             LOG.error("createNATFlowEntries : Unable to create SNAT table entry for fixed ip {}", internalIp);
 505             return;
 506         }
 507         //Install the DNAT default FIB flow L3_FIB_TABLE (21) -> PSNAT_TABLE (26) if SNAT is disabled
 508         boolean isSnatEnabled = NatUtil.isSnatEnabledForRouterId(dataBroker, routerName);
 509         if (!isSnatEnabled) {
 510             addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, writeFlowInvTx, true);
 511         }
 512         //Create the DNAT and SNAT table entries
 513         createDNATTblEntry(dpnId, mapping, routerId, associatedVpnId, writeFlowInvTx);
 514         createSNATTblEntry(dpnId, mapping, vpnId, routerId, associatedVpnId, externalNetworkId, writeFlowInvTx);
 515         floatingIPHandler.onAddFloatingIp(dpnId, routerName, routerId, externalNetworkId, interfaceName, mapping,
 516                 writeFlowInvTx);
 517     }
 518
 519     void createNATOnlyFlowEntries(BigInteger dpnId, String routerName, String associatedVPN,
 520                                   Uuid externalNetworkId, InternalToExternalPortMap mapping) {
 521         String internalIp = mapping.getInternalIp();
 522         //String segmentId = associatedVPN == null ? routerName : associatedVPN;
 523         LOG.debug("createNATOnlyFlowEntries : Retrieving vpn id for VPN {} to proceed with create NAT Flows",
 524                 routerName);
 525         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 526         if (routerId == NatConstants.INVALID_ID) {
 527             LOG.error("createNATOnlyFlowEntries : Could not retrieve vpn id for {} to create NAT Flow entries",
 528                     routerName);
 529             return;
 530         }
 531         long associatedVpnId = NatUtil.getVpnId(dataBroker, associatedVPN);
 532         LOG.debug("createNATOnlyFlowEntries : Associated VPN Id {} for router {}", associatedVpnId, routerName);
 533         long vpnId = getVpnId(externalNetworkId, mapping.getExternalId());
 534         if (vpnId < 0) {
 535             LOG.error("createNATOnlyFlowEntries : Unable to create SNAT table entry for fixed ip {}", internalIp);
 536             return;
 537         }
 538         //Install the DNAT default FIB flow L3_FIB_TABLE (21) -> PSNAT_TABLE (26) if SNAT is disabled
 539         boolean isSnatEnabled = NatUtil.isSnatEnabledForRouterId(dataBroker, routerName);
 540         if (!isSnatEnabled) {
 541             addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, null, true);
 542         }
 543         //Create the DNAT and SNAT table entries
 544         FlowEntity preFlowEntity = buildPreDNATFlowEntity(dpnId, mapping, routerId, associatedVpnId);
 545         mdsalManager.installFlow(preFlowEntity);
 546
 547         FlowEntity flowEntity = buildDNATFlowEntity(dpnId, mapping, routerId, associatedVpnId);
 548         mdsalManager.installFlow(flowEntity);
 549
 550         String externalIp = mapping.getExternalIp();
 551         preFlowEntity = buildPreSNATFlowEntity(dpnId, internalIp, externalIp, vpnId , routerId, associatedVpnId);
 552         mdsalManager.installFlow(preFlowEntity);
 553
 554         flowEntity = buildSNATFlowEntity(dpnId, mapping, vpnId, externalNetworkId);
 555         if (flowEntity != null) {
 556             mdsalManager.installFlow(flowEntity);
 557         }
 558
 559     }
 560
 561     void removeNATFlowEntries(String interfaceName, final InternalToExternalPortMap mapping,
 562                               InstanceIdentifier<RouterPorts> portIid, final String routerName, BigInteger dpnId,
 563                               WriteTransaction removeFlowInvTx) {
 564         String internalIp = mapping.getInternalIp();
 565         String externalIp = mapping.getExternalIp();
 566         //Get the DPN on which this interface resides
 567         if (dpnId == null) {
 568             dpnId = NatUtil.getDpnForInterface(interfaceManager, interfaceName);
 569             if (dpnId.equals(BigInteger.ZERO)) {
 570                 LOG.warn("removeNATFlowEntries: Abort processing Floating ip configuration. No DPN for port: {}",
 571                         interfaceName);
 572                 return;
 573             }
 574         }
 575
 576         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 577         if (routerId == NatConstants.INVALID_ID) {
 578             LOG.error("removeNATFlowEntries : Could not retrieve router id for {} to remove NAT Flow entries",
 579                     routerName);
 580             return;
 581         }
 582
 583         //Delete the DNAT and SNAT table entries
 584         removeDNATTblEntry(dpnId, internalIp, externalIp, routerId, removeFlowInvTx);
 585
 586         Uuid extNwId = getExtNetworkId(portIid, LogicalDatastoreType.OPERATIONAL);
 587         if (extNwId == null) {
 588             LOG.error("removeNATFlowEntries : External network associated with interface {} could not be retrieved",
 589                 interfaceName);
 590             return;
 591         }
 592         long vpnId = getVpnId(extNwId, mapping.getExternalId());
 593         if (vpnId < 0) {
 594             LOG.error("removeNATFlowEntries : No VPN associated with ext nw {}. Unable to delete SNAT table "
 595                 + "entry for fixed ip {}", extNwId, internalIp);
 596             return;
 597         }
 598         removeSNATTblEntry(dpnId, internalIp, externalIp, routerId, vpnId, removeFlowInvTx);
 599         //Remove the DNAT default FIB flow L3_FIB_TABLE (21) -> PSNAT_TABLE (26) if SNAT is disabled
 600         boolean isSnatEnabled = NatUtil.isSnatEnabledForRouterId(dataBroker, routerName);
 601         if (!isSnatEnabled) {
 602             addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, removeFlowInvTx, false);
 603         }
 604         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, extNwId);
 605         if (provType == null) {
 606             LOG.error("removeNATFlowEntries : External Network Provider Type missing");
 607             return;
 608         }
 609         if (provType == ProviderTypes.VXLAN) {
 610             floatingIPHandler.onRemoveFloatingIp(dpnId, routerName, routerId, extNwId, mapping,
 611                     NatConstants.DEFAULT_L3VNI_VALUE, removeFlowInvTx);
 612             removeOperationalDS(routerName, interfaceName, internalIp);
 613             return;
 614         }
 615         long label = getOperationalIpMapping(routerName, interfaceName, internalIp);
 616         if (label < 0) {
 617             LOG.error("removeNATFlowEntries : Could not retrieve label for prefix {} in router {}",
 618                     internalIp, routerId);
 619             return;
 620         }
 621         floatingIPHandler.onRemoveFloatingIp(dpnId, routerName, routerId, extNwId, mapping, (int) label,
 622                 removeFlowInvTx);
 623         removeOperationalDS(routerName, interfaceName, internalIp);
 624     }
 625
 626     void removeNATFlowEntries(BigInteger dpnId, String interfaceName, String vpnName, String routerName,
 627                               InternalToExternalPortMap mapping, WriteTransaction removeFlowInvTx) {
 628         String internalIp = mapping.getInternalIp();
 629         String externalIp = mapping.getExternalIp();
 630         long routerId = NatUtil.getVpnId(dataBroker, routerName);
 631         if (routerId == NatConstants.INVALID_ID) {
 632             LOG.error("removeNATFlowEntries : Could not retrieve router id for {} to remove NAT Flow entries",
 633                     routerName);
 634             return;
 635         }
 636
 637         long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 638         if (vpnId == NatConstants.INVALID_ID) {
 639             LOG.warn("removeNATFlowEntries : VPN Id not found for {} to remove NAT flow entries {}",
 640                     vpnName, internalIp);
 641         }
 642
 643         //Delete the DNAT and SNAT table entries
 644         removeDNATTblEntry(dpnId, internalIp, externalIp, routerId, removeFlowInvTx);
 645         removeSNATTblEntry(dpnId, internalIp, externalIp, routerId, vpnId, removeFlowInvTx);
 646         //Remove the DNAT default FIB flow L3_FIB_TABLE (21) -> PSNAT_TABLE (26) if SNAT is disabled
 647         boolean isSnatEnabled = NatUtil.isSnatEnabledForRouterId(dataBroker, routerName);
 648         if (!isSnatEnabled) {
 649             addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, removeFlowInvTx, false);
 650         }
 651         Uuid externalNetworkId = NatUtil.getNetworkIdFromRouterName(dataBroker,routerName);
 652         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerName, externalNetworkId);
 653         if (provType == null) {
 654             LOG.error("removeNATFlowEntries : External Network Provider Type Missing");
 655             return;
 656         }
 657         if (provType == ProviderTypes.VXLAN) {
 658             floatingIPHandler.cleanupFibEntries(dpnId, vpnName, externalIp, NatConstants.DEFAULT_L3VNI_VALUE,
 659                     removeFlowInvTx, provType);
 660             removeOperationalDS(routerName, interfaceName, internalIp);
 661             return;
 662         }
 663         long label = getOperationalIpMapping(routerName, interfaceName, internalIp);
 664         if (label < 0) {
 665             LOG.error("removeNATFlowEntries : Could not retrieve label for prefix {} in router {}",
 666                     internalIp, routerId);
 667             return;
 668         }
 669         floatingIPHandler.cleanupFibEntries(dpnId, vpnName, externalIp, label, removeFlowInvTx, provType);
 670         removeOperationalDS(routerName, interfaceName, internalIp);
 671     }
 672
 673     protected long getOperationalIpMapping(String routerId, String interfaceName, String internalIp) {
 674         InstanceIdentifier<InternalToExternalPortMap> intExtPortMapIdentifier =
 675             NatUtil.getIntExtPortMapIdentifier(routerId, interfaceName, internalIp);
 676         return SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 677                 LogicalDatastoreType.OPERATIONAL, intExtPortMapIdentifier).toJavaUtil().map(
 678                 InternalToExternalPortMap::getLabel).orElse(NatConstants.INVALID_ID);
 679     }
 680
 681     static void updateOperationalDS(DataBroker dataBroker, String routerId, String interfaceName, long label,
 682                                     String internalIp, String externalIp) {
 683
 684         LOG.info("updateOperationalDS : Updating operational DS for floating ip config : {} with label {}",
 685                 internalIp, label);
 686         InstanceIdentifier<Ports> portsId = NatUtil.getPortsIdentifier(routerId, interfaceName);
 687         Optional<Ports> optPorts =
 688                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 689                         LogicalDatastoreType.OPERATIONAL, portsId);
 690         InternalToExternalPortMap intExtPortMap = new InternalToExternalPortMapBuilder().setKey(new
 691                 InternalToExternalPortMapKey(internalIp)).setInternalIp(internalIp).setExternalIp(externalIp)
 692                 .setLabel(label).build();
 693         if (optPorts.isPresent()) {
 694             LOG.debug("updateOperationalDS : Ports {} entry already present. Updating intExtPortMap for internal ip {}",
 695                     interfaceName, internalIp);
 696             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, portsId.child(InternalToExternalPortMap
 697                     .class, new InternalToExternalPortMapKey(internalIp)), intExtPortMap);
 698         } else {
 699             LOG.debug("updateOperationalDS : Adding Ports entry {} along with intExtPortMap {}",
 700                     interfaceName, internalIp);
 701             List<InternalToExternalPortMap> intExtPortMapList = new ArrayList<>();
 702             intExtPortMapList.add(intExtPortMap);
 703             Ports ports = new PortsBuilder().setKey(new PortsKey(interfaceName)).setPortName(interfaceName)
 704                     .setInternalToExternalPortMap(intExtPortMapList).build();
 705             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, portsId, ports);
 706         }
 707     }
 708
 709     void removeOperationalDS(String routerId, String interfaceName, String internalIp) {
 710         LOG.info("removeOperationalDS : Remove operational DS for floating ip config: {}", internalIp);
 711         InstanceIdentifier<InternalToExternalPortMap> intExtPortMapId = NatUtil.getIntExtPortMapIdentifier(routerId,
 712                 interfaceName, internalIp);
 713         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, intExtPortMapId);
 714     }
 715
 716     private FlowEntity buildPreDNATDeleteFlowEntity(BigInteger dpId, String externalIp, long routerId) {
 717
 718         LOG.info("buildPreDNATDeleteFlowEntity : Bulding Delete DNAT Flow entity for ip {} ", externalIp);
 719
 720         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.PDNAT_TABLE, routerId, externalIp);
 721
 722         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PDNAT_TABLE, flowRef,
 723                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 724                 NwConstants.COOKIE_DNAT_TABLE, null, null);
 725
 726         return flowEntity;
 727     }
 728
 729
 730
 731     private FlowEntity buildDNATDeleteFlowEntity(BigInteger dpId, String internalIp, long routerId) {
 732
 733         LOG.info("buildDNATDeleteFlowEntity : Bulding Delete DNAT Flow entity for ip {} ", internalIp);
 734
 735         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.DNAT_TABLE, routerId, internalIp);
 736
 737         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.DNAT_TABLE, flowRef,
 738                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 739                 NwConstants.COOKIE_DNAT_TABLE, null, null);
 740
 741         return flowEntity;
 742
 743     }
 744
 745     private FlowEntity buildPreSNATDeleteFlowEntity(BigInteger dpId, String internalIp, long routerId) {
 746
 747         LOG.info("buildPreSNATDeleteFlowEntity : Building Delete PSNAT Flow entity for ip {} ", internalIp);
 748
 749         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.PSNAT_TABLE, routerId, internalIp);
 750
 751         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.PSNAT_TABLE, flowRef,
 752                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 753                 NwConstants.COOKIE_DNAT_TABLE, null, null);
 754         return flowEntity;
 755     }
 756
 757     private FlowEntity buildSNATDeleteFlowEntity(BigInteger dpId, String externalIp, long routerId) {
 758
 759         LOG.info("buildSNATDeleteFlowEntity : Building Delete SNAT Flow entity for ip {} ", externalIp);
 760
 761         String flowRef = NatUtil.getFlowRef(dpId, NwConstants.SNAT_TABLE, routerId, externalIp);
 762
 763         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.SNAT_TABLE, flowRef,
 764                 NatConstants.DEFAULT_DNAT_FLOW_PRIORITY, flowRef, 0, 0,
 765                 NwConstants.COOKIE_DNAT_TABLE, null, null);
 766
 767         return flowEntity;
 768     }
 769
 770     private void addOrDelDefaultFibRouteForDnat(BigInteger dpnId, String routerName,
 771                                                 long routerId, WriteTransaction tx, boolean create) {
 772         if (tx == null) {
 773             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 774                 newTx -> addOrDelDefaultFibRouteForDnat(dpnId, routerName, routerId, newTx, create)), LOG,
 775                 "Error handling default FIB route for DNAT");
 776             return;
 777         }
 778         //Check if the router to bgp-vpn association is present
 779         long associatedVpnId = NatConstants.INVALID_ID;
 780         Uuid associatedVpn = NatUtil.getVpnForRouter(dataBroker, routerName);
 781         if (associatedVpn != null) {
 782             associatedVpnId = NatUtil.getVpnId(dataBroker, associatedVpn.getValue());
 783         }
 784         if (create) {
 785             if (associatedVpnId != NatConstants.INVALID_ID) {
 786                 LOG.debug("addOrDelDefaultFibRouteForDnat: Install NAT default route on DPN {} for the router {} with "
 787                         + "vpn-id {}", dpnId, routerName, associatedVpnId);
 788                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, associatedVpnId, routerId, tx);
 789             } else {
 790                 LOG.debug("addOrDelDefaultFibRouteForDnat: Install NAT default route on DPN {} for the router {} with "
 791                         + "vpn-id {}", dpnId, routerName, routerId);
 792                 defaultRouteProgrammer.installDefNATRouteInDPN(dpnId, routerId, tx);
 793             }
 794         } else {
 795             if (associatedVpnId != NatConstants.INVALID_ID) {
 796                 LOG.debug("addOrDelDefaultFibRouteForDnat: Remove NAT default route on DPN {} for the router {} "
 797                         + "with vpn-id {}", dpnId, routerName, associatedVpnId);
 798                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, associatedVpnId, routerId, tx);
 799             } else {
 800                 LOG.debug("addOrDelDefaultFibRouteForDnat: Remove NAT default route on DPN {} for the router {} "
 801                         + "with vpn-id {}", dpnId, routerName, routerId);
 802                 defaultRouteProgrammer.removeDefNATRouteInDPN(dpnId, routerId, tx);
 803             }
 804         }
 805     }
 806 }