vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.Collections;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Objects;
  19 import javax.inject.Inject;
  20 import javax.inject.Singleton;
  21 import org.eclipse.jdt.annotation.NonNull;
  22 import org.eclipse.jdt.annotation.Nullable;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  26 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  28 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  29 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  30 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  31 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  32 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  33 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  34 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  35 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  37 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  57 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  58 import org.opendaylight.yangtools.yang.common.Uint32;
  59 import org.opendaylight.yangtools.yang.common.Uint64;
  60 import org.slf4j.Logger;
  61 import org.slf4j.LoggerFactory;
  62
  63 @Singleton
  64 public class VpnSubnetRouteHandler {
  65     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  66     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  67     private static final String VPN_EVENT_SOURCE_SUBNET_ROUTE = "vpnSubnetRouteEvent";
  68     private final DataBroker dataBroker;
  69     private final SubnetOpDpnManager subOpDpnManager;
  70     private final IBgpManager bgpManager;
  71     private final VpnOpDataSyncer vpnOpDataSyncer;
  72     private final VpnNodeListener vpnNodeListener;
  73     private final IFibManager fibManager;
  74     private final VpnUtil vpnUtil;
  75
  76     @Inject
  77     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  78             final IBgpManager bgpManager, final VpnOpDataSyncer vpnOpDataSyncer, final VpnNodeListener vpnNodeListener,
  79             final IFibManager fibManager, VpnUtil vpnUtil) {
  80         this.dataBroker = dataBroker;
  81         this.subOpDpnManager = subnetOpDpnManager;
  82         this.bgpManager = bgpManager;
  83         this.vpnOpDataSyncer = vpnOpDataSyncer;
  84         this.vpnNodeListener = vpnNodeListener;
  85         this.fibManager = fibManager;
  86         this.vpnUtil = vpnUtil;
  87     }
  88
  89     // TODO Clean up the exception handling
  90     @SuppressWarnings("checkstyle:IllegalCatch")
  91     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  92         Uuid subnetId = subnetmap.getId();
  93         String subnetIp = subnetmap.getSubnetIp();
  94         Subnetmap subMap = null;
  95         SubnetOpDataEntry subOpEntry = null;
  96         SubnetOpDataEntryBuilder subOpBuilder = null;
  97         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
  98         Optional<SubnetOpDataEntry> optionalSubs = null;
  99
 100         Preconditions.checkNotNull(subnetId, LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 101         Preconditions.checkNotNull(subnetIp,
 102                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 103         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 104
 105         if (subnetmap.getVpnId() == null) {
 106             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 107                     subnetId);
 108             return;
 109         }
 110         String vpnName = subnetmap.getVpnId().getValue();
 111         Uint32 vpnId = waitAndGetVpnIdIfInvalid(vpnName);
 112         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 113             LOG.error(
 114                     "{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 115                             + "processing subnet {} with IP {}, bailing out now.",
 116                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 117             return;
 118         }
 119
 120         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 121         VpnInstanceOpDataEntry vpnInstanceOpData = waitAndGetVpnInstanceOpDataIfNull(vpnName, primaryRd);
 122         if (vpnInstanceOpData == null) {
 123             LOG.error(
 124                     "{} onSubnetAddedToVpn: VpnInstanceOpData not yet available for VpnName {} "
 125                             + "processing subnet {} with IP {}, bailing out now.",
 126                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 127             return;
 128         }
 129         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 130                 subnetId.getValue(), subnetIp, vpnName);
 131         //TODO(vivek): Change this to use more granularized lock at subnetId level
 132         try {
 133             vpnUtil.lockSubnet(subnetId.getValue());
 134             // Please check if subnetId belongs to an External Network
 135             InstanceIdentifier<Subnetmap> subMapid =
 136                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 137                             new SubnetmapKey(subnetId)).build();
 138             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 139                     LogicalDatastoreType.CONFIGURATION, subMapid);
 140             if (!sm.isPresent()) {
 141                 LOG.error("{} onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet {} IP {}"
 142                         + " vpnName {}", LOGGING_PREFIX, subnetId, subnetIp, vpnName);
 143                 return;
 144             }
 145             subMap = sm.get();
 146
 147             if (isBgpVpn) {
 148                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 149                         .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 150                 Optional<Networks> optionalNets = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 151                         LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 152                 if (optionalNets.isPresent()) {
 153                     LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 154                                     + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 155                             subnetId.getValue(), subnetIp, subMap.getNetworkId().getValue(), vpnName);
 156                     return;
 157                 }
 158             }
 159             //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 160             subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 161                     new SubnetOpDataEntryKey(subnetId)).build();
 162             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 163                     subOpIdentifier);
 164             if (optionalSubs.isPresent()) {
 165                 LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 166                         + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 167                 return;
 168             }
 169             LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 170                     + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 171             subOpBuilder = new SubnetOpDataEntryBuilder().withKey(new SubnetOpDataEntryKey(subnetId));
 172             subOpBuilder.setSubnetId(subnetId);
 173             subOpBuilder.setSubnetCidr(subnetIp);
 174
 175             if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 176                 LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 177                         + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 178                 return;
 179             }
 180             subOpBuilder.setVrfId(primaryRd);
 181             subOpBuilder.setVpnName(vpnName);
 182             subOpBuilder.setSubnetToDpn(new ArrayList<>());
 183             subOpBuilder.setRouteAdvState(TaskState.Idle);
 184             subOpBuilder.setElanTag(elanTag);
 185             Long l3Vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni().toJava() : 0L;
 186             subOpBuilder.setL3vni(l3Vni);
 187             subOpEntry = subOpBuilder.build();
 188             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 189                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 190             LOG.info("{} onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}", LOGGING_PREFIX,
 191                     subnetId.getValue());
 192         } catch (TransactionCommitFailedException e) {
 193             LOG.error("{} Creation of SubnetOpDataEntry for subnet {} failed ", LOGGING_PREFIX,
 194                     subnetId.getValue(), e);
 195             // The second part of this method depends on subMap being non-null so fail fast here.
 196             return;
 197         } catch (RuntimeException e) { //TODO: Avoid this
 198             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 199                     subnetId.getValue(), subnetIp, vpnName, e);
 200             return;
 201         } catch (ReadFailedException e) {
 202             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 203                     subnetId, subnetIp, vpnName);
 204             return;
 205         } finally {
 206             vpnUtil.unlockSubnet(subnetId.getValue());
 207         }
 208         try {
 209             //In second critical section , Port-Op-Data will be updated.
 210             vpnUtil.lockSubnet(subnetId.getValue());
 211             Uint64 dpnId = null;
 212             SubnetToDpn subDpn = null;
 213             Map<Uint64, SubnetToDpn> subDpnMap = new HashMap<>();
 214
 215             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 216                     subOpIdentifier);
 217             subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get())
 218                     .withKey(new SubnetOpDataEntryKey(subnetId));
 219             List<Uuid> portList = subMap.getPortList();
 220             if (portList != null) {
 221                 for (Uuid port : portList) {
 222                     Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 223                     if (intfState != null) {
 224                         try {
 225                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 226                         } catch (Exception e) {
 227                             LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 228                                             + " subnetroute inclusion for this interface for subnet {} subnetIp {} "
 229                                     + "vpn {} failed with exception", LOGGING_PREFIX, port.getValue(),
 230                                     subnetId.getValue(), subnetIp, vpnName, e);
 231                             continue;
 232                         }
 233                         if (dpnId.equals(Uint64.ZERO)) {
 234                             LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 235                                             + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 236                                     subnetId.getValue(), subnetIp, vpnName);
 237                             continue;
 238                         }
 239                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 240                         if (intfState.getOperStatus() != OperStatus.Up) {
 241                             LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 242                                             + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(),
 243                                     subnetId.getValue(), subnetIp, vpnName);
 244                             continue;
 245                         }
 246                         subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 247                         if (intfState.getOperStatus() == OperStatus.Up) {
 248                             // port is UP
 249                             subDpnMap.put(dpnId, subDpn);
 250                         }
 251                     } else {
 252                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 253                     }
 254                 }
 255                 if (subDpnMap.size() > 0) {
 256                     subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 257                 }
 258             }
 259             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 260                     subMap.getNetworkId(), isBgpVpn);
 261             subOpEntry = subOpBuilder.build();
 262             SingleTransactionDataBroker.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 263                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 264             LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 265                             + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 266                     subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 267                     subOpEntry.getLastAdvState());
 268         } catch (RuntimeException e) {
 269             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 270                     subnetId.getValue(), subnetIp, vpnName, e);
 271         } catch (ReadFailedException e) {
 272             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 273                     subnetId, subnetIp, vpnName);
 274         } catch (TransactionCommitFailedException ex) {
 275             LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {} failed",
 276                     LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 277         } finally {
 278             vpnUtil.unlockSubnet(subnetId.getValue());
 279         }
 280     }
 281
 282     private Uint32 waitAndGetVpnIdIfInvalid(String vpnName) {
 283         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 284         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 285             LOG.debug("VpnId is invalid, waiting to fetch again: vpnName={}, vpnId={}", vpnName, vpnId);
 286             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 287                     VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 288             vpnId = vpnUtil.getVpnId(vpnName);
 289         }
 290         return vpnId;
 291     }
 292
 293     private VpnInstanceOpDataEntry waitAndGetVpnInstanceOpDataIfNull(String vpnName, String primaryRd) {
 294         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 295         if (vpnInstanceOpData == null) {
 296             LOG.debug("vpnInstanceOpData is null, waiting to fetch again: vpnName={}", vpnName);
 297             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnOpData, vpnName,
 298                     VpnConstants.PER_VPN_INSTANCE_OPDATA_MAX_WAIT_TIME_IN_MILLISECONDS);
 299             vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 300         }
 301         return vpnInstanceOpData;
 302     }
 303
 304     // TODO Clean up the exception handling
 305     @SuppressWarnings("checkstyle:IllegalCatch")
 306     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 307         Uuid subnetId = subnetmap.getId();
 308         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 309                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 310         //TODO(vivek): Change this to use more granularized lock at subnetId level
 311         try {
 312             vpnUtil.lockSubnet(subnetId.getValue());
 313             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 314                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 315                             new SubnetOpDataEntryKey(subnetId)).build();
 316             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 317                     LogicalDatastoreType.OPERATIONAL,
 318                     subOpIdentifier);
 319             if (!optionalSubs.isPresent()) {
 320                 LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 321                                 + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 322                         subnetId.getValue(), subnetmap.getVpnId());
 323                 return;
 324             }
 325             LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 326                             + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 327                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 328                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 329                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 330                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 331                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 332                  * going through the list of ports on the subnet
 333                  */
 334             InstanceIdentifier<Subnetmap> subMapid =
 335                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 336                             new SubnetmapKey(subnetId)).build();
 337             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 338                     LogicalDatastoreType.CONFIGURATION, subMapid);
 339             if (!sm.isPresent()) {
 340                 LOG.error("{} onSubnetDeletedFromVpn: Stale ports removal: Unable to retrieve subnetmap entry"
 341                                 + " for subnet {} subnetIp {} vpnName {}", LOGGING_PREFIX, subnetId.getValue(),
 342                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName());
 343             } else {
 344                 Subnetmap subMap = sm.get();
 345                 List<Uuid> portList = subMap.getPortList();
 346                 if (portList != null) {
 347                     for (Uuid port : portList) {
 348                         InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 349                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 350                                         new PortOpDataEntryKey(port.getValue())).build();
 351                         LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 352                                         + " from subnet {} subnetIp {} vpnName {} TaskState {}",
 353                                 LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 354                                 optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 355                                 optionalSubs.get().getRouteAdvState());
 356                         SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 357                                 portOpIdentifier, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 358                     }
 359                 }
 360             }
 361
 362             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 363             String rd = subOpBuilder.getVrfId();
 364             String subnetIp = subOpBuilder.getSubnetCidr();
 365             String vpnName = subOpBuilder.getVpnName();
 366             //Withdraw the routes for all the interfaces on this subnet
 367             //Remove subnet route entry from FIB
 368             deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 369             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 370                     VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 371             LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 372                             + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(),
 373                     subnetIp, vpnName, rd);
 374         } catch (RuntimeException e) { //TODO: Avoid this
 375             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {}",
 376                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 377         } catch (TransactionCommitFailedException ex) {
 378             LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 379                             + " vpnId {} failed", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 380                     subnetmap.getVpnId(), ex);
 381         } catch (ReadFailedException e) {
 382             LOG.error("{} onSubnetDeletedFromVpn: Failed to read data store for subnet {} ip {} vpn {}",
 383                     LOGGING_PREFIX, subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 384         } finally {
 385             vpnUtil.unlockSubnet(subnetId.getValue());
 386         }
 387     }
 388
 389     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 390         Uuid subnetId = subnetmap.getId();
 391         String vpnName = subnetmap.getVpnId().getValue();
 392         String subnetIp = subnetmap.getSubnetIp();
 393
 394         Preconditions.checkNotNull(subnetId,
 395                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 396         Preconditions.checkNotNull(subnetIp,
 397                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 398         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 399         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 400         try {
 401             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 402                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 403                             new SubnetOpDataEntryKey(subnetId)).build();
 404             Optional<SubnetOpDataEntry> optionalSubs =
 405                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 406                             subOpIdentifier);
 407             if (optionalSubs.isPresent()) {
 408                 onSubnetDeletedFromVpn(subnetmap, true);
 409             } else {
 410                 onSubnetAddedToVpn(subnetmap, true, elanTag);
 411             }
 412             LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 413                     subnetId.getValue(), subnetIp, vpnName);
 414         } catch (ReadFailedException e) {
 415             LOG.error("onSubnetUpdatedInVpn: Failed to read data store for subnet{} ip {} elanTag {} vpn {}",subnetId,
 416                     subnetIp, elanTag, vpnName);
 417         }
 418     }
 419
 420     // TODO Clean up the exception handling
 421     @SuppressWarnings("checkstyle:IllegalCatch")
 422     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 423         Uuid subnetId = subnetmap.getId();
 424         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 425                 subnetId.getValue());
 426         //TODO(vivek): Change this to use more granularized lock at subnetId level
 427         try {
 428             vpnUtil.lockSubnet(subnetId.getValue());
 429             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 430                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 431                             new SubnetOpDataEntryKey(subnetId)).build();
 432
 433             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 434                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 435             if (!optionalSubs.isPresent()) {
 436                 LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 437                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 438                 return;
 439             }
 440             String vpnName = optionalSubs.get().getVpnName();
 441             String subnetIp = optionalSubs.get().getSubnetCidr();
 442             String rd = optionalSubs.get().getVrfId();
 443             String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 444             LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 445                             + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 446                     vpnName, rd, routeAdvState);
 447             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 448             Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 449             if (intfState == null) {
 450                 // Interface State not yet available
 451                 return;
 452             }
 453             final Uint64 dpnId;
 454             try {
 455                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 456             } catch (Exception e) {
 457                 LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 458                                 + " for this interface failed for subnet {} subnetIp {} vpn {} rd {}",
 459                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd, e);
 460                 return;
 461             }
 462             if (dpnId.equals(Uint64.ZERO)) {
 463                 LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 464                                 + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 465                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 466                 return;
 467             }
 468             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 469             if (intfState.getOperStatus() != OperStatus.Up) {
 470                 LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 471                                 + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 472                         portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 473                 return;
 474             }
 475             LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 476                             + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 477                     subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 478             SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 479             if (subDpn == null) {
 480                 LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 481                                 + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 482                         portId.getValue(), vpnName, rd, subnetIp);
 483                 return;
 484             }
 485             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 486             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 487             subOpBuilder.setSubnetToDpn(concat(subnetOpDataEntry.getSubnetToDpn(), subDpn));
 488             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 489                 if (subOpBuilder.getNhDpnId() == null) {
 490                     // No nexthop selected yet, elect one now
 491                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 492                             subnetmap.getNetworkId(), true);
 493                 } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 494                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 495                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 496                 }
 497             }
 498             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 499             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 500                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 501             LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 502                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 503                     portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 504                     subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 505         } catch (RuntimeException e) { //TODO: Avoid this
 506             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {}", LOGGING_PREFIX,
 507                     portId.getValue(), subnetId.getValue(), e);
 508         } catch (ReadFailedException e) {
 509             LOG.error("{} onPortAddedToSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 510                     portId, subnetId);
 511         } catch (TransactionCommitFailedException e) {
 512             LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied",
 513                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), e);
 514         } finally {
 515             vpnUtil.unlockSubnet(subnetId.getValue());
 516         }
 517     }
 518
 519     // TODO Clean up the exception handling
 520     @SuppressWarnings("checkstyle:IllegalCatch")
 521     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 522         Uuid subnetId = subnetmap.getId();
 523         //TODO(vivek): Change this to use more granularized lock at subnetId level
 524         try {
 525             vpnUtil.lockSubnet(subnetId.getValue());
 526             PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 527                     subnetmap.getId());
 528             if (portOpEntry == null) {
 529                 return;
 530             }
 531             Uint64 dpnId = portOpEntry.getDpnId();
 532             if (dpnId == null) {
 533                 LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 534                                 + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 535                         subnetId.getValue());
 536                 return;
 537             }
 538             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 539             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 540                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 541                             new SubnetOpDataEntryKey(subnetId)).build();
 542             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 543                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 544             if (!optionalSubs.isPresent()) {
 545                 LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 546                         + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 547                 return;
 548             }
 549             LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 550                             + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 551                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 552                     optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 553                     optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 554             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 555             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 556             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 557             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 558                 // select another NhDpnId
 559                 if (last) {
 560                     LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 561                                     + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 562                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 563                     // last port on this DPN, so we need to elect the new NHDpnId
 564                     electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 565                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 566                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 567                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 568                     LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 569                                     + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 570                             portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 571                             subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 572                 }
 573             }
 574         } catch (RuntimeException e) {
 575             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {}", LOGGING_PREFIX,
 576                     portId.getValue(), subnetId.getValue(), e);
 577         } catch (ReadFailedException e) {
 578             LOG.error("{} onPortRemovedFromSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 579                     portId, subnetId);
 580         } catch (TransactionCommitFailedException e) {
 581             LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed", LOGGING_PREFIX,
 582                     portId.getValue(), subnetId.getValue(), e);
 583         } finally {
 584             vpnUtil.unlockSubnet(subnetId.getValue());
 585         }
 586     }
 587
 588     // TODO Clean up the exception handling
 589     @SuppressWarnings("checkstyle:IllegalCatch")
 590     public void onInterfaceUp(Uint64 dpnId, String intfName, Uuid subnetId) {
 591         //TODO(vivek): Change this to use more granularized lock at subnetId level
 592         SubnetToDpn subDpn = null;
 593         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 594             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 595                     intfName, subnetId.getValue());
 596             return;
 597         }
 598         try {
 599             vpnUtil.lockSubnet(subnetId.getValue());
 600             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 601                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 602                             new SubnetOpDataEntryKey(subnetId)).build();
 603             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 604                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 605             if (!optionalSubs.isPresent()) {
 606                 LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 607                         + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 608                 return;
 609             }
 610             subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 611             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 612             if (subDpn == null) {
 613                 return;
 614             }
 615             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 616             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 617             LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 618                             + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 619                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 620                     subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 621             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 622                     subnetId.getValue());
 623             subOpBuilder.setSubnetToDpn(concat(subnetOpDataEntry.getSubnetToDpn(), subDpn));
 624             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 625                 if (subOpBuilder.getNhDpnId() == null) {
 626                     // No nexthop selected yet, elect one now
 627                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 628                             null /*networkId*/, !isExternalSubnetVpn);
 629                 } else if (!isExternalSubnetVpn) {
 630                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 631                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 632                 }
 633             }
 634             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 635             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 636                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 637             LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 638                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 639                     subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 640                     subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 641         } catch (RuntimeException e) {
 642             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {}",
 643                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 644         } catch (ReadFailedException e) {
 645             LOG.error("{} onInterfaceUp: Failed to read data store for interface {} dpn {} subnet {}", LOGGING_PREFIX,
 646                     intfName, dpnId, subnetId);
 647         } catch (TransactionCommitFailedException e) {
 648             LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed",
 649                     LOGGING_PREFIX, subnetId.getValue(), intfName, e);
 650         } finally {
 651             vpnUtil.unlockSubnet(subnetId.getValue());
 652         }
 653     }
 654
 655     // TODO Clean up the exception handling
 656     @SuppressWarnings("checkstyle:IllegalCatch")
 657     public void onInterfaceDown(final Uint64 dpnId, final String interfaceName, Uuid subnetId) {
 658         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 659             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 660                     interfaceName, subnetId.getValue());
 661             return;
 662         }
 663         try {
 664             vpnUtil.lockSubnet(subnetId.getValue());
 665             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 666             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 667                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 668                             new SubnetOpDataEntryKey(subnetId)).build();
 669             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 670                     LogicalDatastoreType.OPERATIONAL,
 671                     subOpIdentifier);
 672             if (!optionalSubs.isPresent()) {
 673                 LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 674                         + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 675                 return;
 676             }
 677             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 678             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 679             LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 680                             + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 681                     subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 682                     subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 683                     interfaceName);
 684             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 685             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 686                 // select another NhDpnId
 687                 if (last) {
 688                     LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 689                                     + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 690                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 691                     // last port on this DPN, so we need to elect the new NHDpnId
 692                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 693                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 694                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 695                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 696                     LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 697                                     + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 698                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 699                             interfaceName);
 700                 }
 701             }
 702         } catch (RuntimeException e) { //TODO: Remove RuntimeException
 703             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {}",
 704                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 705         } catch (ReadFailedException e) {
 706             LOG.error("{} onInterfaceDown: Failed to read data store for interface {} dpn {} subnet {}",
 707                     LOGGING_PREFIX, interfaceName, dpnId, subnetId.getValue(), e);
 708         } catch (TransactionCommitFailedException ex) {
 709             LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {} failed",
 710                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 711         } finally {
 712             vpnUtil.unlockSubnet(subnetId.getValue());
 713         }
 714     }
 715
 716     // TODO Clean up the exception handling
 717     @SuppressWarnings("checkstyle:IllegalCatch")
 718     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, Uint64 dpnId) {
 719         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 720                 dpnId.toString());
 721         try {
 722             vpnUtil.lockSubnet(subnetId.getValue());
 723             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 724                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 725                             new SubnetOpDataEntryKey(subnetId)).build();
 726             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 727                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 728             if (!optionalSubs.isPresent()) {
 729                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 730                         LOGGING_PREFIX, subnetId.getValue());
 731                 return;
 732             }
 733             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 734                             + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 735                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 736                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 737                     optionalSubs.get().getLastAdvState(), dpnId.toString());
 738             SubnetOpDataEntry subOpEntry = optionalSubs.get();
 739             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 740             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 741             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 742                 if (subOpBuilder.getNhDpnId() == null) {
 743                     // No nexthop selected yet, elect one now
 744                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 745                             null /*networkId*/, !isExternalSubnetVpn);
 746                 } else if (!isExternalSubnetVpn) {
 747                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 748                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 749                 }
 750             }
 751             subOpEntry = subOpBuilder.build();
 752             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 753                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 754             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 755                             + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 756                     LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 757                     subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 758                     subOpEntry.getLastAdvState());
 759         } catch (RuntimeException e) { //TODO: lockSubnet() throws this exception. Rectify lockSubnet()
 760             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}",
 761                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 762         } catch (TransactionCommitFailedException ex) {
 763             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to update subnetRoute for subnet {} on dpn {}",
 764                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 765         } catch (ReadFailedException e) {
 766             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to read data store for subnet {} on dpn {}",
 767                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 768         }
 769         finally {
 770             vpnUtil.unlockSubnet(subnetId.getValue());
 771         }
 772     }
 773
 774     // TODO Clean up the exception handling
 775     @SuppressWarnings("checkstyle:IllegalCatch")
 776     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, Uint64 dpnId) {
 777         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 778         //TODO(vivek): Change this to use more granularized lock at subnetId level
 779         try {
 780             vpnUtil.lockSubnet(subnetId.getValue());
 781             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 782                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 783                             new SubnetOpDataEntryKey(subnetId)).build();
 784             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 785                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 786             if (!optionalSubs.isPresent()) {
 787                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 788                         + " is not available", LOGGING_PREFIX, subnetId.getValue());
 789                 return;
 790             }
 791             LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 792                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 793                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 794                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 795                     optionalSubs.get().getLastAdvState());
 796             SubnetOpDataEntry subOpEntry = null;
 797             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 798             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 799             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 800                 electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 801                 subOpEntry = subOpBuilder.build();
 802                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 803                         subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 804                 LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 805                                 + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 806                         dpnId.toString(), optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 807                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 808                         optionalSubs.get().getLastAdvState());
 809             }
 810         } catch (RuntimeException e) {
 811             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 812                     + " dpnId {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 813         } catch (ReadFailedException e) {
 814             LOG.error("{} Failed to read data store for subnet {} dpn {}", LOGGING_PREFIX, subnetId, dpnId);
 815         } catch (TransactionCommitFailedException e) {
 816             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 817                     + " on dpn {} failed", LOGGING_PREFIX, subnetId.getValue(), dpnId, e);
 818         } finally {
 819             vpnUtil.unlockSubnet(subnetId.getValue());
 820         }
 821     }
 822
 823     @SuppressWarnings("checkstyle:IllegalCatch")
 824     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 825         try {
 826             //BGP manager will handle withdraw and advertise internally if prefix
 827             //already exist
 828             Uint32 label = Uint32.ZERO;
 829             Uint32 l3vni = Uint32.ZERO;
 830
 831             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 832             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 833                 l3vni = subOpBuilder.getL3vni();
 834             } else {
 835                 label = subOpBuilder.getLabel();
 836                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
 837                     LOG.error("publishSubnetRouteToBgp: Label not found for rd {}, subnetIp {}",
 838                             subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr());
 839                     return;
 840                 }
 841             }
 842             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 843                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 844                     Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 845             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 846         } catch (Exception e) {
 847             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 848                     + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 849                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 850         }
 851     }
 852
 853     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 854         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 855                 subOpBuilder.getNhDpnId());
 856         if (nhTepIp != null) {
 857             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 858         } else {
 859             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 860                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 861                     subOpBuilder.getNhDpnId().toString());
 862             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 863         }
 864     }
 865
 866     // TODO Clean up the exception handling
 867     @SuppressWarnings("checkstyle:IllegalCatch")
 868     private boolean addSubnetRouteToFib(String rd, String subnetIp, Uint64 nhDpnId, String nextHopIp,
 869                                         String vpnName, Long elanTag, Uint32 label, Uint32 l3vni,
 870                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 871
 872         Preconditions.checkNotNull(rd,
 873                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 874         Preconditions.checkNotNull(subnetIp,
 875                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 876         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 877         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 878         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 879         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 880         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 881         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 882                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 883         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 884                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni.longValue())
 885                 .setLabel(label.longValue()).setElanTag(elanTag)
 886                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 887         if (!isBgpVpn) {
 888             vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 889             return true;
 890         }
 891         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 892         vpnUtil.syncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil
 893                 .getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(vpnName), subnetIp), VpnUtil
 894                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, Prefixes.PrefixCue.SubnetRoute));
 895         vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 896         try {
 897             // BGP manager will handle withdraw and advertise internally if prefix
 898             // already exist
 899             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 900                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 901         } catch (Exception e) {
 902             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 903                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 904             return false;
 905         }
 906         return true;
 907     }
 908
 909     private Uint32 getLabel(String rd, String subnetIp) {
 910         Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME, VpnUtil.getNextHopLabelKey(rd, subnetIp));
 911         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 912                 subnetIp);
 913         return label;
 914     }
 915
 916     // TODO Clean up the exception handling
 917     @SuppressWarnings("checkstyle:IllegalCatch")
 918     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 919         Preconditions.checkNotNull(rd,
 920                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 921         Preconditions.checkNotNull(subnetIp,
 922                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 923         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 924         if (isBgpVpn) {
 925             try {
 926                 bgpManager.withdrawPrefix(rd, subnetIp);
 927             } catch (Exception e) {
 928                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}",
 929                         LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 930                 return false;
 931             }
 932         }
 933         return true;
 934     }
 935
 936     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 937         fibManager.removeFibEntry(rd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 938         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 939         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 940             String importingRd = vpnInstance.getVrfId();
 941             fibManager.removeFibEntry(importingRd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 942             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 943                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 944         }
 945         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 946                 + " vpnName {}", prefix, rd, vpnName);
 947     }
 948
 949     // TODO Clean up the exception handling
 950     @SuppressWarnings("checkstyle:IllegalCatch")
 951     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, @Nullable Uint64 oldDpnId,
 952                                            Uuid subnetId, Uuid networkId, boolean isBgpVpn) {
 953         boolean isRouteAdvertised = false;
 954         String rd = subOpBuilder.getVrfId();
 955         String subnetIp = subOpBuilder.getSubnetCidr();
 956         String vpnName = subOpBuilder.getVpnName();
 957         long elanTag = subOpBuilder.getElanTag().toJava();
 958         boolean isAlternateDpnSelected = false;
 959         Uint32 l3vni = Uint32.ZERO;
 960         Uint32 label = Uint32.ZERO;
 961         String networkName = networkId != null ? networkId.getValue() : null;
 962
 963         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} TaskState {}"
 964                 + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 965                 subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 966         if (!isBgpVpn) {
 967             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 968             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 969             // blended more better into this design.
 970             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 971                 l3vni = subOpBuilder.getL3vni();
 972             } else {
 973                 label = getLabel(rd, subnetIp);
 974                 subOpBuilder.setLabel(label);
 975             }
 976             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 977                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 978             if (isRouteAdvertised) {
 979                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 980             } else {
 981                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 982                     + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 983                     vpnName, rd);
 984                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 985             }
 986             return;
 987         }
 988
 989         String nhTepIp = null;
 990         Uint64 nhDpnId = null;
 991         List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 992         if (subDpnList != null) {
 993             for (SubnetToDpn subnetToDpn : subDpnList) {
 994                 if (subnetToDpn.getDpnId().equals(oldDpnId)) {
 995                     // Is this same is as input dpnId, then ignore it
 996                     continue;
 997                 }
 998                 nhDpnId = subnetToDpn.getDpnId();
 999                 if (vpnNodeListener.isConnectedNode(nhDpnId)) {
1000                     // selected dpnId is connected to ODL
1001                     // but does it have a TEP configured at all?
1002                     try {
1003                         nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
1004                         if (nhTepIp != null) {
1005                             isAlternateDpnSelected = true;
1006                             break;
1007                         }
1008                     } catch (Exception e) {
1009                         LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
1010                                 + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
1011                     }
1012                 }
1013             }
1014         }
1015         if (!isAlternateDpnSelected) {
1016             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
1017             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
1018                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
1019                         + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName,
1020                         rd);
1021                 // Withdraw route from BGP for this subnet
1022                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
1023                 //subOpBuilder.setNhDpnId(Uint64.valueOf(null));
1024                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1025                 if (routeWithdrawn) {
1026                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
1027                 } else {
1028                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
1029                         + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
1030                         subnetIp, vpnName, rd);
1031                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
1032                 }
1033             }
1034         } else {
1035             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
1036             subOpBuilder.setNhDpnId(nhDpnId);
1037             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
1038                 l3vni = subOpBuilder.getL3vni();
1039             } else {
1040                 label = getLabel(rd, subnetIp);
1041                 subOpBuilder.setLabel(label);
1042             }
1043             //update the VRF entry for the subnetroute.
1044             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
1045                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
1046             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1047             if (isRouteAdvertised) {
1048                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1049             } else {
1050                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1051                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1052                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1053             }
1054         }
1055     }
1056
1057     private static boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1058         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1059                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1060     }
1061
1062     private static @NonNull List<SubnetToDpn> concat(@Nullable List<SubnetToDpn> list, @NonNull SubnetToDpn entry) {
1063         final List<SubnetToDpn> ret = new ArrayList<>();
1064         if (list != null) {
1065             ret.addAll(list);
1066         }
1067         ret.add(entry);
1068         return ret;
1069     }
1070 }
1071