vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Objects;
  21 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  22 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  23 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  24 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  25 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  26 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  27 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  28 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  29 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  30 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  31 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  32 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  52 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 public class VpnSubnetRouteHandler {
  57     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  58     private final DataBroker dataBroker;
  59     private final SubnetOpDpnManager subOpDpnManager;
  60     private final IBgpManager bgpManager;
  61     private final VpnInterfaceManager vpnInterfaceManager;
  62     private final IdManagerService idManager;
  63     private LockManagerService lockManager;
  64     private final VpnOpDataSyncer vpnOpDataSyncer;
  65     private final VpnNodeListener vpnNodeListener;
  66
  67     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  68         final IBgpManager bgpManager, final VpnInterfaceManager vpnIntfManager, final IdManagerService idManager,
  69         LockManagerService lockManagerService, final VpnOpDataSyncer vpnOpDataSyncer,
  70         final VpnNodeListener vpnNodeListener) {
  71         this.dataBroker = dataBroker;
  72         this.subOpDpnManager = subnetOpDpnManager;
  73         this.bgpManager = bgpManager;
  74         this.vpnInterfaceManager = vpnIntfManager;
  75         this.idManager = idManager;
  76         this.lockManager = lockManagerService;
  77         this.vpnOpDataSyncer = vpnOpDataSyncer;
  78         this.vpnNodeListener = vpnNodeListener;
  79     }
  80
  81     // TODO Clean up the exception handling
  82     @SuppressWarnings("checkstyle:IllegalCatch")
  83     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  84         Uuid subnetId = subnetmap.getId();
  85         String subnetIp = subnetmap.getSubnetIp();
  86         boolean isRouteAdvertised = false;
  87
  88         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  89         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  90         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  91
  92         LOG.info("onSubnetAddedToVpn: Subnet {} being added to vpn", subnetId.getValue());
  93
  94         String vpnName;
  95         if (subnetmap.getVpnId() != null) {
  96             vpnName = subnetmap.getVpnId().getValue();
  97             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
  98             if (vpnId == VpnConstants.INVALID_ID) {
  99                 vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 100                         VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 101                 vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 102                 if (vpnId == VpnConstants.INVALID_ID) {
 103                     LOG.error("onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 104                               + "processing subnet {} with IP {}, bailing out now.", vpnName, subnetId, subnetIp);
 105                     return;
 106                 }
 107             }
 108         } else {
 109             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 110                       subnetId);
 111             return;
 112         }
 113
 114         //TODO(vivek): Change this to use more granularized lock at subnetId level
 115         try {
 116             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 117             try {
 118                 Subnetmap subMap = null;
 119
 120                 // Please check if subnetId belongs to an External Network
 121                 InstanceIdentifier<Subnetmap> subMapid =
 122                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 123                         new SubnetmapKey(subnetId)).build();
 124                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 125                 if (!sm.isPresent()) {
 126                     LOG.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 127                     return;
 128                 }
 129                 subMap = sm.get();
 130
 131                 if (isBgpVpn) {
 132                     InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 133                             .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 134                     Optional<Networks> optionalNets = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
 135                             netsIdentifier);
 136                     if (optionalNets.isPresent()) {
 137                         LOG.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network" + " {},"
 138                                 + " so ignoring this" + " for SubnetRoute",
 139                                 subnetId.getValue(), subMap.getNetworkId().getValue());
 140                         return;
 141                     }
 142                 }
 143                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 144                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 145                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 146                         new SubnetOpDataEntryKey(subnetId)).build();
 147                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 148                         LogicalDatastoreType.OPERATIONAL,
 149                         subOpIdentifier);
 150                 if (optionalSubs.isPresent()) {
 151                     LOG.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} already detected to be present",
 152                         subnetId.getValue());
 153                     return;
 154                 }
 155                 LOG.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 156                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<>();
 157                 BigInteger dpnId = null;
 158                 SubnetToDpn subDpn = null;
 159                 SubnetOpDataEntryBuilder subOpBuilder =
 160                     new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 161                 subOpBuilder.setSubnetId(subnetId);
 162                 subOpBuilder.setSubnetCidr(subnetIp);
 163                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 164
 165                 if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 166                     LOG.error("onSubnetAddedToVpn: The VPN Instance name " + vpnName + " does not have RD ");
 167                     return;
 168                 }
 169
 170                 subOpBuilder.setVrfId(primaryRd);
 171                 subOpBuilder.setVpnName(vpnName);
 172                 subOpBuilder.setSubnetToDpn(new ArrayList<>());
 173                 subOpBuilder.setRouteAdvState(TaskState.Idle);
 174                 subOpBuilder.setElanTag(elanTag);
 175                 Long l3Vni = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd).getL3vni();
 176                 subOpBuilder.setL3vni(l3Vni);
 177
 178                 // First recover set of ports available in this subnet
 179                 List<Uuid> portList = subMap.getPortList();
 180                 if (portList != null) {
 181                     for (Uuid port : portList) {
 182                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 183                         if (intfState != null) {
 184                             try {
 185                                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 186                             } catch (Exception e) {
 187                                 LOG.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 188                                         " subnetroute inclusion for this interface failed with exception {}",
 189                                         port.getValue(), e);
 190                                 continue;
 191                             }
 192                             if (dpnId.equals(BigInteger.ZERO)) {
 193                                 LOG.info("onSubnetAddedToVpn: Port " + port.getValue()
 194                                     + " is not assigned DPN yet, ignoring ");
 195                                 continue;
 196                             }
 197                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 198                             if (intfState.getOperStatus() != OperStatus.Up) {
 199                                 LOG.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 200                                 continue;
 201                             }
 202                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 203                             if (intfState.getOperStatus() == OperStatus.Up) {
 204                                 // port is UP
 205                                 subDpnMap.put(dpnId, subDpn);
 206                             }
 207                         } else {
 208                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 209                         }
 210                     }
 211                     if (subDpnMap.size() > 0) {
 212                         subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 213                     }
 214                 }
 215                 electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 216                         subMap.getNetworkId(), isBgpVpn);
 217                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 218                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 219                 LOG.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}",
 220                         subnetId.getValue());
 221             } catch (Exception ex) {
 222                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 223             } finally {
 224                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 225             }
 226         } catch (Exception e) {
 227             LOG.error("Unable to handle subnet {} added to vpn {} {}", subnetIp, vpnName, e);
 228         }
 229     }
 230
 231     // TODO Clean up the exception handling
 232     @SuppressWarnings("checkstyle:IllegalCatch")
 233     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 234         Uuid subnetId = subnetmap.getId();
 235         LOG.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
 236         //TODO(vivek): Change this to use more granularized lock at subnetId level
 237         try {
 238             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 239             try {
 240                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 241                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 242                         new SubnetOpDataEntryKey(subnetId)).build();
 243                 LOG.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 244                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 245                         LogicalDatastoreType.OPERATIONAL,
 246                         subOpIdentifier);
 247                 if (!optionalSubs.isPresent()) {
 248                     LOG.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} not available in datastore",
 249                         subnetId.getValue());
 250                     return;
 251                 }
 252
 253                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 254                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 255                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 256                  * going through the list of ports on the subnet
 257                  */
 258                 InstanceIdentifier<Subnetmap> subMapid =
 259                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 260                         new SubnetmapKey(subnetId)).build();
 261                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 262                 if (!sm.isPresent()) {
 263                     LOG.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 264                 } else {
 265                     Subnetmap subMap = sm.get();
 266                     List<Uuid> portList = subMap.getPortList();
 267                     if (portList != null) {
 268                         for (Uuid port : portList) {
 269                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 270                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 271                                     new PortOpDataEntryKey(port.getValue())).build();
 272                             LOG.trace("Deleting portOpData entry for port " + port.getValue());
 273                             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 274                         }
 275                     }
 276                 }
 277
 278                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 279                 String rd = subOpBuilder.getVrfId();
 280                 String subnetIp = subOpBuilder.getSubnetCidr();
 281                 String vpnName = subOpBuilder.getVpnName();
 282                 //Withdraw the routes for all the interfaces on this subnet
 283                 //Remove subnet route entry from FIB
 284                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 285                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 286                 LOG.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore",
 287                     subnetId.getValue());
 288             } catch (Exception ex) {
 289                 LOG.error("Removal of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 290             } finally {
 291                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 292             }
 293         } catch (Exception e) {
 294             LOG.error("Unable to handle subnet {} removed to vpn {}", subnetmap.getSubnetIp(),
 295                 subnetmap.getVpnId().getValue(), e);
 296         }
 297     }
 298
 299     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, boolean oldVpnType,
 300                                      boolean newVpnType, Long elanTag) {
 301         Uuid subnetId = subnetmap.getId();
 302         String vpnName = subnetmap.getVpnId().getValue();
 303         String subnetIp = subnetmap.getSubnetIp();
 304
 305         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 306         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 307         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 308         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 309
 310         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 311             InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 312                 new SubnetOpDataEntryKey(subnetId)).build();
 313         Optional<SubnetOpDataEntry> optionalSubs =
 314             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 315         if (optionalSubs.isPresent()) {
 316             onSubnetDeletedFromVpn(subnetmap, oldVpnType);
 317         } else {
 318             onSubnetAddedToVpn(subnetmap, newVpnType, elanTag);
 319         }
 320     }
 321
 322     // TODO Clean up the exception handling
 323     @SuppressWarnings("checkstyle:IllegalCatch")
 324     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 325         Uuid subnetId = subnetmap.getId();
 326         boolean isRouteAdvertised = false;
 327         LOG.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 328         //TODO(vivek): Change this to use more granularized lock at subnetId level
 329         try {
 330             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 331             try {
 332                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 333                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 334                         new SubnetOpDataEntryKey(subnetId)).build();
 335
 336                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 337                         subOpIdentifier);
 338                 if (!optionalSubs.isPresent()) {
 339                     LOG.info("onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 340                         portId.getValue(), subnetId.getValue());
 341                     return;
 342                 }
 343                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 344                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 345                 if (intfState == null) {
 346                     // Interface State not yet available
 347                     return;
 348                 }
 349                 BigInteger dpnId = BigInteger.ZERO;
 350                 try {
 351                     dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 352                 } catch (Exception e) {
 353                     LOG.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 354                             " subnetroute inclusion for this interface failed with exception {}",
 355                             portId.getValue(), e);
 356                     return;
 357                 }
 358                 if (dpnId.equals(BigInteger.ZERO)) {
 359                     LOG.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 360                     return;
 361                 }
 362                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 363                 if (intfState.getOperStatus() != OperStatus.Up) {
 364                     LOG.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 365                     return;
 366                 }
 367                 LOG.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet {}",
 368                     subnetId.getValue());
 369                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 370                 if (subDpn == null) {
 371                     return;
 372                 }
 373                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 374                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 375                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 376                 subDpnList.add(subDpn);
 377                 subOpBuilder.setSubnetToDpn(subDpnList);
 378                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 379                     if (subOpBuilder.getNhDpnId() == null) {
 380                         // No nexthop selected yet, elect one now
 381                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 382                                 subnetmap.getNetworkId(), true);
 383                     } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 384                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 385                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 386                     }
 387                 }
 388                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 389                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 390                 LOG.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {}",
 391                     portId.getValue());
 392             } catch (Exception ex) {
 393                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 394             } finally {
 395                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 396             }
 397         } catch (Exception e) {
 398             LOG.error("Unable to handle port {} added to subnet {} {}", portId.getValue(), subnetId.getValue(), e);
 399         }
 400     }
 401
 402     // TODO Clean up the exception handling
 403     @SuppressWarnings("checkstyle:IllegalCatch")
 404     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 405         Uuid subnetId = subnetmap.getId();
 406         boolean isRouteAdvertised = false;
 407
 408         LOG.info(
 409             "onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 410         //TODO(vivek): Change this to use more granularized lock at subnetId level
 411         try {
 412             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 413             try {
 414                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 415                 if (portOpEntry == null) {
 416                     return;
 417                 }
 418                 BigInteger dpnId = portOpEntry.getDpnId();
 419                 if (dpnId == null) {
 420                     LOG.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring",
 421                         portId.getValue());
 422                     return;
 423                 }
 424                 LOG.debug(
 425                     "onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 426                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 427                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 428                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 429                         new SubnetOpDataEntryKey(subnetId)).build();
 430                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 431                     subOpIdentifier);
 432                 if (!optionalSubs.isPresent()) {
 433                     LOG.info("onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 434                         portId.getValue(), subnetId.getValue());
 435                     return;
 436                 }
 437                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 438                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 439                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 440                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 441                     // select another NhDpnId
 442                     if (last) {
 443                         LOG.debug("onPortRemovedFromSubnet: Last port {} on the subnet {}", portId,
 444                             subnetId.getValue());
 445                         // last port on this DPN, so we need to elect the new NHDpnId
 446                         electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 447                                 !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 448                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 449                                 subOpBuilder.build());
 450                         LOG.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port "
 451                                 + portId.getValue());
 452                     }
 453                 }
 454             } catch (Exception ex) {
 455                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 456             } finally {
 457                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 458             }
 459         } catch (Exception e) {
 460             LOG.error("Unable to handle port {} removed from subnet {} {}", portId.getValue(), subnetId.getValue(),
 461                 e);
 462         }
 463     }
 464
 465     // TODO Clean up the exception handling
 466     @SuppressWarnings("checkstyle:IllegalCatch")
 467     public void onInterfaceUp(BigInteger dpnId, String intfName, Uuid subnetId) {
 468         LOG.info("onInterfaceUp: Port " + intfName);
 469         //TODO(vivek): Change this to use more granularized lock at subnetId level
 470         SubnetToDpn subDpn = null;
 471         if ((dpnId == null) || Objects.equals(dpnId, BigInteger.ZERO)) {
 472             LOG.error("onInterfaceUp: Unable to determine the DPNID for port {}" + intfName);
 473             return;
 474         }
 475         try {
 476             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 477             try {
 478                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 479                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 480                         new SubnetOpDataEntryKey(subnetId)).build();
 481                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 482                     subOpIdentifier);
 483                 if (!optionalSubs.isPresent()) {
 484                     LOG.error("onInterfaceUp: SubnetOpDataEntry for subnet {} is not available", subnetId.getValue());
 485                     return;
 486                 }
 487
 488                 LOG.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 489                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 490                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 491                 if (subDpn == null) {
 492                     return;
 493                 }
 494                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 495                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 496                 boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 497                         subnetId.getValue());
 498                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 499                 subDpnList.add(subDpn);
 500                 subOpBuilder.setSubnetToDpn(subDpnList);
 501                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 502                     if (subOpBuilder.getNhDpnId() == null) {
 503                         // No nexthop selected yet, elect one now
 504                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 505                                 null /*networkId*/, !isExternalSubnetVpn);
 506                     } else if (!isExternalSubnetVpn) {
 507                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 508                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 509                     }
 510                 }
 511                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 512                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 513                 LOG.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 514             } catch (Exception ex) {
 515                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 516             } finally {
 517                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 518             }
 519         } catch (Exception e) {
 520             LOG.error("Unable to handle interface up event for port {} in subnet {} {}", intfName,
 521                 subnetId.getValue(), e);
 522         }
 523     }
 524
 525     // TODO Clean up the exception handling
 526     @SuppressWarnings("checkstyle:IllegalCatch")
 527     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName, Uuid subnetId) {
 528         LOG.info("onInterfaceDown: Port " + interfaceName);
 529         if ((dpnId == null) || (Objects.equals(dpnId, BigInteger.ZERO))) {
 530             LOG.error("onInterfaceDown: Unable to determine the DPNID for port " + interfaceName);
 531             return;
 532         }
 533         try {
 534             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 535             try {
 536                 LOG.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 537                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 538                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 539                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 540                         new SubnetOpDataEntryKey(subnetId)).build();
 541                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 542                     LogicalDatastoreType.OPERATIONAL,
 543                     subOpIdentifier);
 544                 if (!optionalSubs.isPresent()) {
 545                     LOG.error("onInterfaceDown: SubnetOpDataEntry for subnet {} is not available", subnetId.getValue());
 546                     return;
 547                 }
 548                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 549                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 550                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 551                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 552                     // select another NhDpnId
 553                     if (last) {
 554                         LOG.debug(
 555                             "onInterfaceDown: Last active port " + interfaceName + " on the subnet: " + subnetId
 556                                 .getValue());
 557                         // last port on this DPN, so we need to elect the new NHDpnId
 558                         electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 559                                 !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 560                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 561                                 subOpBuilder.build());
 562                         LOG.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore for port {}",
 563                                 interfaceName);
 564                     }
 565                 }
 566             } catch (Exception ex) {
 567                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 568             } finally {
 569                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 570             }
 571         } catch (Exception e) {
 572             LOG.error("Unable to handle interface down event for port {} in subnet {} {}", interfaceName,
 573                 subnetId.getValue(), e);
 574         }
 575     }
 576
 577     // TODO Clean up the exception handling
 578     @SuppressWarnings("checkstyle:IllegalCatch")
 579     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, BigInteger dpnId) {
 580         boolean isRouteAdvertised = false;
 581         LOG.info("updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 582         try {
 583             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 584             try {
 585                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 586                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 587                         new SubnetOpDataEntryKey(subnetId)).build();
 588                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 589                     LogicalDatastoreType.OPERATIONAL,
 590                     subOpIdentifier);
 591                 if (!optionalSubs.isPresent()) {
 592                     LOG.error("updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 593                         subnetId.getValue());
 594                     return;
 595                 }
 596                 SubnetOpDataEntry subOpEntry = optionalSubs.get();
 597                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 598                 boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 599                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 600                     if (subOpBuilder.getNhDpnId() == null) {
 601                         // No nexthop selected yet, elect one now
 602                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 603                                 null /*networkId*/, !isExternalSubnetVpn);
 604                     } else if (!isExternalSubnetVpn) {
 605                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 606                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 607                     }
 608                 }
 609                 subOpEntry = subOpBuilder.build();
 610                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 611                 LOG.info(
 612                     "updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up on dpn"
 613                         + " {} for subnet {}",
 614                     dpnId.toString(), subnetId.getValue());
 615             } catch (Exception ex) {
 616                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 617             } finally {
 618                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 619             }
 620         } catch (Exception e) {
 621             LOG.error("Unable to handle tunnel up event for subnetId {} dpnId {}", subnetId.getValue(),
 622                 dpnId.toString());
 623         }
 624     }
 625
 626     // TODO Clean up the exception handling
 627     @SuppressWarnings("checkstyle:IllegalCatch")
 628     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, BigInteger dpnId) {
 629         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 630         //TODO(vivek): Change this to use more granularized lock at subnetId level
 631         try {
 632             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 633             try {
 634                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 635                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 636                         new SubnetOpDataEntryKey(subnetId)).build();
 637                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 638                     LogicalDatastoreType.OPERATIONAL,
 639                     subOpIdentifier);
 640                 if (!optionalSubs.isPresent()) {
 641                     LOG.error("updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {} is not available",
 642                         subnetId.getValue());
 643                     return;
 644                 }
 645                 SubnetOpDataEntry subOpEntry = null;
 646                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 647                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 648                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 649                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 650                     subOpEntry = subOpBuilder.build();
 651                     MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 652                     LOG.info(
 653                         "updateSubnetRouteOnTunnelDownEvent: Updated subnetopdataentry to OP Datastore tunnnel down "
 654                             + "on dpn {} for subnet {}",
 655                         dpnId.toString(), subnetId.getValue());
 656                 }
 657             } catch (Exception ex) {
 658                 LOG.error("Updation of SubnetOpDataEntry for subnet {} failed", subnetId.getValue(), ex);
 659             } finally {
 660                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 661             }
 662         } catch (Exception e) {
 663             LOG.error("Unable to handle tunnel down event for subnetId {} dpnId {}", subnetId.getValue(),
 664                 dpnId.toString());
 665         }
 666     }
 667
 668     @SuppressWarnings("checkstyle:IllegalCatch")
 669     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 670         try {
 671             //BGP manager will handle withdraw and advertise internally if prefix
 672             //already exist
 673             long label = 0;
 674             long l3vni = 0;
 675             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 676             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 677                 l3vni = subOpBuilder.getL3vni();
 678             } else {
 679                 label = subOpBuilder.getLabel();
 680             }
 681             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 682                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 683                     0 /*l2vni*/, null /*gatewayMacAddress*/);
 684             subOpBuilder.setRouteAdvState(TaskState.Advertised);
 685         } catch (Exception e) {
 686             LOG.error("Fail: Subnet route not advertised for rd {} subnetIp {} with dpnid {}",
 687                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(), nextHopIp, e);
 688         }
 689     }
 690
 691     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 692         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 693                 subOpBuilder.getNhDpnId());
 694         if (nhTepIp != null) {
 695             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 696         } else {
 697             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 698                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 699                     subOpBuilder.getNhDpnId().toString());
 700             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 701         }
 702     }
 703
 704     // TODO Clean up the exception handling
 705     @SuppressWarnings("checkstyle:IllegalCatch")
 706     private boolean addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String nextHopIp,
 707                                         String vpnName, Long elanTag, long label, long l3vni,
 708                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 709
 710         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 711         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 712         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 713         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 714         Preconditions.checkNotNull(label, "label cannot be null or empty!");
 715         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 716         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 717         LOG.info("Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 718                 vpnName, subnetIp, elanTag);
 719         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 720                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni).setLabel(label).setElanTag(elanTag)
 721                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 722         if (!isBgpVpn) {
 723             vpnPopulator.populateFib(input, null /*writeCfgTxn*/, null /*writeOperTxn*/);
 724             return true;
 725         }
 726         Preconditions.checkNotNull(nextHopIp, "NextHopIp cannot be null or empty!");
 727         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, VpnUtil
 728                 .getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, vpnName), subnetIp), VpnUtil
 729                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, subnetId, true /*isNatPrefix*/));
 730         vpnPopulator.populateFib(input, null /*writeCfgTxn*/, null /*writeOperTxn*/);
 731         try {
 732             // BGP manager will handle withdraw and advertise internally if prefix
 733             // already exist
 734             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 735                     encapType, label, l3vni, 0 /*l2vni*/, null /*gatewayMacAddress*/);
 736         } catch (Exception e) {
 737             LOG.error("Fail: Subnet route not advertised for rd {} subnetIp {} with dpnId {}", rd, subnetIp, e,
 738                     nhDpnId, e);
 739             return false;
 740         }
 741         return true;
 742     }
 743
 744     private int getLabel(String rd, String subnetIp) {
 745         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 746             VpnUtil.getNextHopLabelKey(rd, subnetIp));
 747         LOG.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
 748         return label;
 749     }
 750
 751     // TODO Clean up the exception handling
 752     @SuppressWarnings("checkstyle:IllegalCatch")
 753     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 754         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 755         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 756         vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 757         if (isBgpVpn) {
 758             try {
 759                 bgpManager.withdrawPrefix(rd, subnetIp);
 760             } catch (Exception e) {
 761                 LOG.error("Fail: Subnet route not withdrawn for rd {} subnetIp {} due to exception {}",
 762                         rd, subnetIp, e);
 763                 return false;
 764             }
 765         }
 766         return true;
 767     }
 768
 769     // TODO Clean up the exception handling
 770     @SuppressWarnings("checkstyle:IllegalCatch")
 771     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, BigInteger oldDpnId, Uuid subnetId,
 772                                            Uuid networkId, boolean isBgpVpn) {
 773         List<SubnetToDpn> subDpnList = null;
 774         boolean isRouteAdvertised = false;
 775         subDpnList = subOpBuilder.getSubnetToDpn();
 776         String rd = subOpBuilder.getVrfId();
 777         String subnetIp = subOpBuilder.getSubnetCidr();
 778         String vpnName = subOpBuilder.getVpnName();
 779         long elanTag = subOpBuilder.getElanTag();
 780         BigInteger nhDpnId = null;
 781         String nhTepIp = null;
 782         boolean isAlternateDpnSelected = false;
 783         Iterator<SubnetToDpn> subnetDpnIter = subDpnList.iterator();
 784         long l3vni = 0;
 785         long label = 0;
 786         if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 787             l3vni = subOpBuilder.getL3vni();
 788         } else {
 789             label = getLabel(rd, subnetIp);
 790             subOpBuilder.setLabel(label);
 791         }
 792         if (!isBgpVpn) {
 793             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 794             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 795             // blended more better into this design.
 796             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
 797                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkId.getValue());
 798             if (isRouteAdvertised) {
 799                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 800             } else {
 801                 LOG.error("electNewDpnForSubnetRoute: Adding FibEntry for rd {} subnet {} failed",
 802                         rd, subnetId.getValue());
 803                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 804             }
 805             return;
 806         }
 807         while (subnetDpnIter.hasNext()) {
 808             SubnetToDpn subnetToDpn = subnetDpnIter.next();
 809             if (subnetToDpn.getDpnId().equals(oldDpnId)) {
 810                 // Is this same is as input dpnId, then ignore it
 811                 continue;
 812             }
 813             nhDpnId = subnetToDpn.getDpnId();
 814             if (vpnNodeListener.isConnectedNode(nhDpnId)) {
 815                 // selected dpnId is connected to ODL
 816                 // but does it have a TEP configured at all?
 817                 try {
 818                     nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 819                     if (nhTepIp != null) {
 820                         isAlternateDpnSelected = true;
 821                         break;
 822                     }
 823                 } catch (Exception e) {
 824                     LOG.warn("Unable to find TepIp for rd {} subnetroute subnetip {} for dpnid {}, attempt next",
 825                             rd, subnetIp, nhDpnId.toString());
 826                     continue;
 827                 }
 828             }
 829         }
 830         if (!isAlternateDpnSelected) {
 831             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
 832             if (isRouteAdvertised(subOpBuilder) && (oldDpnId != null)) {
 833                 LOG.info("electNewDpnForSubnetRoute: No alternate DPN available for subnet {}."
 834                         + "  Prefix withdrawn from BGP", subnetIp);
 835                 // Withdraw route from BGP for this subnet
 836                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 837                 subOpBuilder.setNhDpnId(null);
 838                 if (routeWithdrawn) {
 839                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
 840                 } else {
 841                     LOG.error("electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {}"
 842                             + " from BGP failed", oldDpnId.toString(), subnetId.getValue());
 843                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
 844                 }
 845             }
 846         } else {
 847             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
 848             subOpBuilder.setNhDpnId(nhDpnId);
 849             //update the VRF entry for the subnetroute.
 850             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
 851                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkId.getValue());
 852             if (isRouteAdvertised) {
 853                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 854             } else {
 855                 LOG.error("electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for rd {} subnet {} failed",
 856                         nhDpnId, rd, subnetId.getValue());
 857                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 858             }
 859         }
 860     }
 861
 862     private boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
 863         return ((subOpBuilder.getRouteAdvState() == TaskState.Advertised)
 864                 || (subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise));
 865     }
 866 }
 867