vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Preconditions;
  11 import java.util.ArrayList;
  12 import java.util.Arrays;
  13 import java.util.Collections;
  14 import java.util.HashMap;
  15 import java.util.List;
  16 import java.util.Map;
  17 import java.util.Objects;
  18 import java.util.Optional;
  19 import java.util.Set;
  20 import java.util.concurrent.ExecutionException;
  21 import java.util.stream.Collectors;
  22 import javax.inject.Inject;
  23 import javax.inject.Singleton;
  24 import org.eclipse.jdt.annotation.NonNull;
  25 import org.eclipse.jdt.annotation.Nullable;
  26 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  27 import org.opendaylight.mdsal.binding.api.DataBroker;
  28 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  29 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  30 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  31 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  32 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  33 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  34 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  35 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  36 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  37 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  39 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpnKey;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  61 import org.opendaylight.yangtools.yang.common.Uint32;
  62 import org.opendaylight.yangtools.yang.common.Uint64;
  63 import org.slf4j.Logger;
  64 import org.slf4j.LoggerFactory;
  65
  66 @Singleton
  67 public class VpnSubnetRouteHandler {
  68     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  69     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  70     private static final String VPN_EVENT_SOURCE_SUBNET_ROUTE = "vpnSubnetRouteEvent";
  71     private final DataBroker dataBroker;
  72     private final SubnetOpDpnManager subOpDpnManager;
  73     private final IBgpManager bgpManager;
  74     private final VpnOpDataSyncer vpnOpDataSyncer;
  75     private final VpnNodeListener vpnNodeListener;
  76     private final IFibManager fibManager;
  77     private final VpnUtil vpnUtil;
  78
  79     @Inject
  80     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  81             final IBgpManager bgpManager, final VpnOpDataSyncer vpnOpDataSyncer, final VpnNodeListener vpnNodeListener,
  82             final IFibManager fibManager, VpnUtil vpnUtil) {
  83         this.dataBroker = dataBroker;
  84         this.subOpDpnManager = subnetOpDpnManager;
  85         this.bgpManager = bgpManager;
  86         this.vpnOpDataSyncer = vpnOpDataSyncer;
  87         this.vpnNodeListener = vpnNodeListener;
  88         this.fibManager = fibManager;
  89         this.vpnUtil = vpnUtil;
  90     }
  91
  92     // TODO Clean up the exception handling
  93     @SuppressWarnings("checkstyle:IllegalCatch")
  94     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  95         Uuid subnetId = subnetmap.getId();
  96         String subnetIp = subnetmap.getSubnetIp();
  97         Subnetmap subMap = null;
  98         SubnetOpDataEntry subOpEntry = null;
  99         SubnetOpDataEntryBuilder subOpBuilder = null;
 100         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
 101         Optional<SubnetOpDataEntry> optionalSubs = null;
 102
 103         Preconditions.checkNotNull(subnetId, LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 104         Preconditions.checkNotNull(subnetIp,
 105                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 106         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 107
 108         if (subnetmap.getVpnId() == null) {
 109             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 110                     subnetId);
 111             return;
 112         }
 113         String vpnName = subnetmap.getVpnId().getValue();
 114         Uint32 vpnId = waitAndGetVpnIdIfInvalid(vpnName);
 115         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 116             LOG.error(
 117                     "{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 118                             + "processing subnet {} with IP {}, bailing out now.",
 119                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 120             return;
 121         }
 122
 123         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 124         VpnInstanceOpDataEntry vpnInstanceOpData = waitAndGetVpnInstanceOpDataIfNull(vpnName, primaryRd);
 125         if (vpnInstanceOpData == null) {
 126             LOG.error(
 127                     "{} onSubnetAddedToVpn: VpnInstanceOpData not yet available for VpnName {} "
 128                             + "processing subnet {} with IP {}, bailing out now.",
 129                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 130             return;
 131         }
 132         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 133                 subnetId.getValue(), subnetIp, vpnName);
 134         //TODO(vivek): Change this to use more granularized lock at subnetId level
 135         try {
 136             vpnUtil.lockSubnet(subnetId.getValue());
 137             // Please check if subnetId belongs to an External Network
 138             InstanceIdentifier<Subnetmap> subMapid =
 139                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 140                             new SubnetmapKey(subnetId)).build();
 141             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 142                     LogicalDatastoreType.CONFIGURATION, subMapid);
 143             if (!sm.isPresent()) {
 144                 LOG.error("{} onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet {} IP {}"
 145                         + " vpnName {}", LOGGING_PREFIX, subnetId, subnetIp, vpnName);
 146                 return;
 147             }
 148             subMap = sm.get();
 149
 150             if (isBgpVpn) {
 151                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 152                         .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 153                 Optional<Networks> optionalNets = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 154                         LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 155                 if (optionalNets.isPresent()) {
 156                     LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 157                                     + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 158                             subnetId.getValue(), subnetIp, subMap.getNetworkId().getValue(), vpnName);
 159                     return;
 160                 }
 161             }
 162             //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 163             subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 164                     new SubnetOpDataEntryKey(subnetId)).build();
 165             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 166                     subOpIdentifier);
 167             if (optionalSubs.isPresent()) {
 168                 LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 169                         + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 170                 return;
 171             }
 172             LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 173                     + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 174             subOpBuilder = new SubnetOpDataEntryBuilder().withKey(new SubnetOpDataEntryKey(subnetId));
 175             subOpBuilder.setSubnetId(subnetId);
 176             subOpBuilder.setSubnetCidr(subnetIp);
 177
 178             if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 179                 LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 180                         + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 181                 return;
 182             }
 183             subOpBuilder.setVrfId(primaryRd);
 184             subOpBuilder.setVpnName(vpnName);
 185             subOpBuilder.setSubnetToDpn(new ArrayList<>());
 186             subOpBuilder.setRouteAdvState(TaskState.Idle);
 187             subOpBuilder.setElanTag(elanTag);
 188             Long l3Vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni().toJava() : 0L;
 189             subOpBuilder.setL3vni(l3Vni);
 190             subOpEntry = subOpBuilder.build();
 191             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 192                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 193             LOG.info("{} onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}", LOGGING_PREFIX,
 194                     subnetId.getValue());
 195         } catch (TransactionCommitFailedException e) {
 196             LOG.error("{} Creation of SubnetOpDataEntry for subnet {} failed ", LOGGING_PREFIX,
 197                     subnetId.getValue(), e);
 198             // The second part of this method depends on subMap being non-null so fail fast here.
 199             return;
 200         } catch (RuntimeException e) { //TODO: Avoid this
 201             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 202                     subnetId.getValue(), subnetIp, vpnName, e);
 203             return;
 204         } catch (InterruptedException | ExecutionException e) {
 205             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 206                     subnetId, subnetIp, vpnName);
 207             return;
 208         } finally {
 209             vpnUtil.unlockSubnet(subnetId.getValue());
 210         }
 211         try {
 212             //In second critical section , Port-Op-Data will be updated.
 213             vpnUtil.lockSubnet(subnetId.getValue());
 214             Uint64 dpnId = null;
 215             SubnetToDpn subDpn = null;
 216             Map<Uint64, SubnetToDpn> subDpnMap = new HashMap<>();
 217
 218             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 219                     subOpIdentifier);
 220             subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get())
 221                     .withKey(new SubnetOpDataEntryKey(subnetId));
 222             List<Uuid> portList = subMap.getPortList();
 223             if (portList != null) {
 224                 for (Uuid port : portList) {
 225                     Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 226                     if (intfState != null) {
 227                         try {
 228                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 229                         } catch (Exception e) {
 230                             LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 231                                             + " subnetroute inclusion for this interface for subnet {} subnetIp {} "
 232                                     + "vpn {} failed with exception", LOGGING_PREFIX, port.getValue(),
 233                                     subnetId.getValue(), subnetIp, vpnName, e);
 234                             continue;
 235                         }
 236                         if (dpnId.equals(Uint64.ZERO)) {
 237                             LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 238                                             + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 239                                     subnetId.getValue(), subnetIp, vpnName);
 240                             continue;
 241                         }
 242                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 243                         if (intfState.getOperStatus() != OperStatus.Up) {
 244                             LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 245                                             + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(),
 246                                     subnetId.getValue(), subnetIp, vpnName);
 247                             continue;
 248                         }
 249                         subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 250                         if (intfState.getOperStatus() == OperStatus.Up) {
 251                             // port is UP
 252                             subDpnMap.put(dpnId, subDpn);
 253                         }
 254                     } else {
 255                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 256                     }
 257                 }
 258                 if (subDpnMap.size() > 0) {
 259                     subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 260                 }
 261             }
 262             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 263                     subMap.getNetworkId(), isBgpVpn);
 264             subOpEntry = subOpBuilder.build();
 265             SingleTransactionDataBroker.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 266                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 267             LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 268                             + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 269                     subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 270                     subOpEntry.getLastAdvState());
 271         } catch (RuntimeException e) {
 272             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 273                     subnetId.getValue(), subnetIp, vpnName, e);
 274         } catch (InterruptedException | ExecutionException e) {
 275             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 276                     subnetId, subnetIp, vpnName);
 277         } catch (TransactionCommitFailedException ex) {
 278             LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {} failed",
 279                     LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 280         } finally {
 281             vpnUtil.unlockSubnet(subnetId.getValue());
 282         }
 283     }
 284
 285     private Uint32 waitAndGetVpnIdIfInvalid(String vpnName) {
 286         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 287         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 288             LOG.debug("VpnId is invalid, waiting to fetch again: vpnName={}, vpnId={}", vpnName, vpnId);
 289             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 290                     VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 291             vpnId = vpnUtil.getVpnId(vpnName);
 292         }
 293         return vpnId;
 294     }
 295
 296     private VpnInstanceOpDataEntry waitAndGetVpnInstanceOpDataIfNull(String vpnName, String primaryRd) {
 297         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 298         if (vpnInstanceOpData == null) {
 299             LOG.debug("vpnInstanceOpData is null, waiting to fetch again: vpnName={}", vpnName);
 300             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnOpData, vpnName,
 301                     VpnConstants.PER_VPN_INSTANCE_OPDATA_MAX_WAIT_TIME_IN_MILLISECONDS);
 302             vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 303         }
 304         return vpnInstanceOpData;
 305     }
 306
 307     // TODO Clean up the exception handling
 308     @SuppressWarnings("checkstyle:IllegalCatch")
 309     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 310         Uuid subnetId = subnetmap.getId();
 311         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 312                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 313         //TODO(vivek): Change this to use more granularized lock at subnetId level
 314         try {
 315             vpnUtil.lockSubnet(subnetId.getValue());
 316             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 317                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 318                             new SubnetOpDataEntryKey(subnetId)).build();
 319             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 320                     LogicalDatastoreType.OPERATIONAL,
 321                     subOpIdentifier);
 322             if (!optionalSubs.isPresent()) {
 323                 LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 324                                 + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 325                         subnetId.getValue(), subnetmap.getVpnId());
 326                 return;
 327             }
 328             LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 329                             + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 330                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 331                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 332                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 333                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 334                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 335                  * going through the list of ports on the subnet
 336                  */
 337             InstanceIdentifier<Subnetmap> subMapid =
 338                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 339                             new SubnetmapKey(subnetId)).build();
 340             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 341                     LogicalDatastoreType.CONFIGURATION, subMapid);
 342             if (!sm.isPresent()) {
 343                 LOG.error("{} onSubnetDeletedFromVpn: Stale ports removal: Unable to retrieve subnetmap entry"
 344                                 + " for subnet {} subnetIp {} vpnName {}", LOGGING_PREFIX, subnetId.getValue(),
 345                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName());
 346             } else {
 347                 Subnetmap subMap = sm.get();
 348                 List<Uuid> portList = subMap.getPortList();
 349                 if (portList != null) {
 350                     for (Uuid port : portList) {
 351                         InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 352                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 353                                         new PortOpDataEntryKey(port.getValue())).build();
 354                         LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 355                                         + " from subnet {} subnetIp {} vpnName {} TaskState {}",
 356                                 LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 357                                 optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 358                                 optionalSubs.get().getRouteAdvState());
 359                         SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 360                                 portOpIdentifier, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 361                     }
 362                 }
 363             }
 364
 365             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 366             String rd = subOpBuilder.getVrfId();
 367             String subnetIp = subOpBuilder.getSubnetCidr();
 368             String vpnName = subOpBuilder.getVpnName();
 369             //Withdraw the routes for all the interfaces on this subnet
 370             //Remove subnet route entry from FIB
 371             deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 372             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 373                     VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 374             LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 375                             + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(),
 376                     subnetIp, vpnName, rd);
 377         } catch (RuntimeException e) { //TODO: Avoid this
 378             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {}",
 379                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 380         } catch (TransactionCommitFailedException ex) {
 381             LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 382                             + " vpnId {} failed", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 383                     subnetmap.getVpnId(), ex);
 384         } catch (InterruptedException | ExecutionException e) {
 385             LOG.error("{} onSubnetDeletedFromVpn: Failed to read data store for subnet {} ip {} vpn {}",
 386                     LOGGING_PREFIX, subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 387         } finally {
 388             vpnUtil.unlockSubnet(subnetId.getValue());
 389         }
 390     }
 391
 392     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 393         Uuid subnetId = subnetmap.getId();
 394         String vpnName = subnetmap.getVpnId().getValue();
 395         String subnetIp = subnetmap.getSubnetIp();
 396
 397         Preconditions.checkNotNull(subnetId,
 398                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 399         Preconditions.checkNotNull(subnetIp,
 400                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 401         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 402         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 403         try {
 404             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 405                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 406                             new SubnetOpDataEntryKey(subnetId)).build();
 407             Optional<SubnetOpDataEntry> optionalSubs =
 408                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 409                             subOpIdentifier);
 410             if (optionalSubs.isPresent()) {
 411                 onSubnetDeletedFromVpn(subnetmap, true);
 412             } else {
 413                 onSubnetAddedToVpn(subnetmap, true, elanTag);
 414             }
 415             LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 416                     subnetId.getValue(), subnetIp, vpnName);
 417         } catch (InterruptedException | ExecutionException e) {
 418             LOG.error("onSubnetUpdatedInVpn: Failed to read data store for subnet{} ip {} elanTag {} vpn {}",subnetId,
 419                     subnetIp, elanTag, vpnName);
 420         }
 421     }
 422
 423     // TODO Clean up the exception handling
 424     @SuppressWarnings("checkstyle:IllegalCatch")
 425     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 426         Uuid subnetId = subnetmap.getId();
 427         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 428                 subnetId.getValue());
 429         //TODO(vivek): Change this to use more granularized lock at subnetId level
 430         try {
 431             vpnUtil.lockSubnet(subnetId.getValue());
 432             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 433                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 434                             new SubnetOpDataEntryKey(subnetId)).build();
 435
 436             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 437                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 438             if (!optionalSubs.isPresent()) {
 439                 LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 440                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 441                 return;
 442             }
 443             String vpnName = optionalSubs.get().getVpnName();
 444             String subnetIp = optionalSubs.get().getSubnetCidr();
 445             String rd = optionalSubs.get().getVrfId();
 446             String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 447             LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 448                             + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 449                     vpnName, rd, routeAdvState);
 450             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 451             Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 452             if (intfState == null) {
 453                 // Interface State not yet available
 454                 return;
 455             }
 456             final Uint64 dpnId;
 457             try {
 458                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 459             } catch (Exception e) {
 460                 LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 461                                 + " for this interface failed for subnet {} subnetIp {} vpn {} rd {}",
 462                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd, e);
 463                 return;
 464             }
 465             if (dpnId.equals(Uint64.ZERO)) {
 466                 LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 467                                 + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 468                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 469                 return;
 470             }
 471             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 472             if (intfState.getOperStatus() != OperStatus.Up) {
 473                 LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 474                                 + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 475                         portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 476                 return;
 477             }
 478             LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 479                             + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 480                     subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 481             SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 482             if (subDpn == null) {
 483                 LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 484                                 + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 485                         portId.getValue(), vpnName, rd, subnetIp);
 486                 return;
 487             }
 488             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 489             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 490             List<SubnetToDpn> subnetToDpnList = concat(new ArrayList<SubnetToDpn>(subnetOpDataEntry
 491                     .nonnullSubnetToDpn().values()), subDpn);
 492             subOpBuilder.setSubnetToDpn(getSubnetToDpnMap(subnetToDpnList));
 493             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 494                 if (subOpBuilder.getNhDpnId() == null) {
 495                     // No nexthop selected yet, elect one now
 496                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 497                             subnetmap.getNetworkId(), true);
 498                 } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 499                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 500                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 501                 }
 502             }
 503             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 504             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 505                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 506             LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 507                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 508                     portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 509                     subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 510         } catch (RuntimeException e) { //TODO: Avoid this
 511             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {}", LOGGING_PREFIX,
 512                     portId.getValue(), subnetId.getValue(), e);
 513         } catch (InterruptedException | ExecutionException e) {
 514             LOG.error("{} onPortAddedToSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 515                     portId, subnetId);
 516         } catch (TransactionCommitFailedException e) {
 517             LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied",
 518                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), e);
 519         } finally {
 520             vpnUtil.unlockSubnet(subnetId.getValue());
 521         }
 522     }
 523
 524     // TODO Clean up the exception handling
 525     @SuppressWarnings("checkstyle:IllegalCatch")
 526     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 527         Uuid subnetId = subnetmap.getId();
 528         //TODO(vivek): Change this to use more granularized lock at subnetId level
 529         try {
 530             vpnUtil.lockSubnet(subnetId.getValue());
 531             PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 532                     subnetmap.getId());
 533             if (portOpEntry == null) {
 534                 return;
 535             }
 536             Uint64 dpnId = portOpEntry.getDpnId();
 537             if (dpnId == null) {
 538                 LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 539                                 + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 540                         subnetId.getValue());
 541                 return;
 542             }
 543             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 544             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 545                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 546                             new SubnetOpDataEntryKey(subnetId)).build();
 547             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 548                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 549             if (!optionalSubs.isPresent()) {
 550                 LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 551                         + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 552                 return;
 553             }
 554             LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 555                             + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 556                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 557                     optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 558                     optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 559             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 560             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 561             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 562             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 563                 // select another NhDpnId
 564                 if (last) {
 565                     LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 566                                     + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 567                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 568                     // last port on this DPN, so we need to elect the new NHDpnId
 569                     electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 570                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 571                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 572                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 573                     LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 574                                     + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 575                             portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 576                             subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 577                 }
 578             }
 579         } catch (RuntimeException e) {
 580             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {}", LOGGING_PREFIX,
 581                     portId.getValue(), subnetId.getValue(), e);
 582         } catch (InterruptedException | ExecutionException e) {
 583             LOG.error("{} onPortRemovedFromSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 584                     portId, subnetId);
 585         } catch (TransactionCommitFailedException e) {
 586             LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed", LOGGING_PREFIX,
 587                     portId.getValue(), subnetId.getValue(), e);
 588         } finally {
 589             vpnUtil.unlockSubnet(subnetId.getValue());
 590         }
 591     }
 592
 593     // TODO Clean up the exception handling
 594     @SuppressWarnings("checkstyle:IllegalCatch")
 595     public void onInterfaceUp(Uint64 dpnId, String intfName, Uuid subnetId) {
 596         //TODO(vivek): Change this to use more granularized lock at subnetId level
 597         SubnetToDpn subDpn = null;
 598         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 599             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 600                     intfName, subnetId.getValue());
 601             return;
 602         }
 603         try {
 604             vpnUtil.lockSubnet(subnetId.getValue());
 605             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 606                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 607                             new SubnetOpDataEntryKey(subnetId)).build();
 608             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 609                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 610             if (!optionalSubs.isPresent()) {
 611                 LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 612                         + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 613                 return;
 614             }
 615             subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 616             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 617             if (subDpn == null) {
 618                 return;
 619             }
 620             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 621             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 622             LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 623                             + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 624                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 625                     subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 626             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 627                     subnetId.getValue());
 628             List<SubnetToDpn> subnetToDpnList = concat(new ArrayList<SubnetToDpn>(subnetOpDataEntry
 629                     .nonnullSubnetToDpn().values()), subDpn);
 630             subOpBuilder.setSubnetToDpn(getSubnetToDpnMap(subnetToDpnList));
 631             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 632                 if (subOpBuilder.getNhDpnId() == null) {
 633                     // No nexthop selected yet, elect one now
 634                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 635                             null /*networkId*/, !isExternalSubnetVpn);
 636                 } else if (!isExternalSubnetVpn) {
 637                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 638                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 639                 }
 640             }
 641             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 642             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 643                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 644             LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 645                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 646                     subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 647                     subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 648         } catch (RuntimeException e) {
 649             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {}",
 650                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 651         } catch (InterruptedException | ExecutionException e) {
 652             LOG.error("{} onInterfaceUp: Failed to read data store for interface {} dpn {} subnet {}", LOGGING_PREFIX,
 653                     intfName, dpnId, subnetId);
 654         } catch (TransactionCommitFailedException e) {
 655             LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed",
 656                     LOGGING_PREFIX, subnetId.getValue(), intfName, e);
 657         } finally {
 658             vpnUtil.unlockSubnet(subnetId.getValue());
 659         }
 660     }
 661
 662     private Map<SubnetToDpnKey, SubnetToDpn> getSubnetToDpnMap(List<SubnetToDpn> subnetToDpnList) {
 663         //convert to set to remove duplicates.
 664         Set<SubnetToDpn> subnetToDpnSet = subnetToDpnList.stream().collect(Collectors.toSet());
 665         Map<SubnetToDpnKey, SubnetToDpn> subnetToDpnMap = new HashMap<>();
 666         for (SubnetToDpn subnetToDpn : subnetToDpnSet) {
 667             subnetToDpnMap.put(new SubnetToDpnKey(subnetToDpn.key()), subnetToDpn);
 668         }
 669         return subnetToDpnMap;
 670     }
 671
 672     // TODO Clean up the exception handling
 673     @SuppressWarnings("checkstyle:IllegalCatch")
 674     public void onInterfaceDown(final Uint64 dpnId, final String interfaceName, Uuid subnetId) {
 675         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 676             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 677                     interfaceName, subnetId.getValue());
 678             return;
 679         }
 680         try {
 681             vpnUtil.lockSubnet(subnetId.getValue());
 682             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 683             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 684                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 685                             new SubnetOpDataEntryKey(subnetId)).build();
 686             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 687                     LogicalDatastoreType.OPERATIONAL,
 688                     subOpIdentifier);
 689             if (!optionalSubs.isPresent()) {
 690                 LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 691                         + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 692                 return;
 693             }
 694             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 695             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 696             LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 697                             + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 698                     subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 699                     subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 700                     interfaceName);
 701             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 702             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 703                 // select another NhDpnId
 704                 if (last) {
 705                     LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 706                                     + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 707                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 708                     // last port on this DPN, so we need to elect the new NHDpnId
 709                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 710                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 711                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 712                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 713                     LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 714                                     + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 715                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 716                             interfaceName);
 717                 }
 718             }
 719         } catch (RuntimeException e) { //TODO: Remove RuntimeException
 720             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {}",
 721                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 722         } catch (InterruptedException | ExecutionException e) {
 723             LOG.error("{} onInterfaceDown: Failed to read data store for interface {} dpn {} subnet {}",
 724                     LOGGING_PREFIX, interfaceName, dpnId, subnetId.getValue(), e);
 725         } catch (TransactionCommitFailedException ex) {
 726             LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {} failed",
 727                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 728         } finally {
 729             vpnUtil.unlockSubnet(subnetId.getValue());
 730         }
 731     }
 732
 733     // TODO Clean up the exception handling
 734     @SuppressWarnings("checkstyle:IllegalCatch")
 735     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, Uint64 dpnId) {
 736         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 737                 dpnId.toString());
 738         try {
 739             vpnUtil.lockSubnet(subnetId.getValue());
 740             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 741                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 742                             new SubnetOpDataEntryKey(subnetId)).build();
 743             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 744                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 745             if (!optionalSubs.isPresent()) {
 746                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 747                         LOGGING_PREFIX, subnetId.getValue());
 748                 return;
 749             }
 750             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 751                             + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 752                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 753                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 754                     optionalSubs.get().getLastAdvState(), dpnId.toString());
 755             SubnetOpDataEntry subOpEntry = optionalSubs.get();
 756             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 757             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 758             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 759                 if (subOpBuilder.getNhDpnId() == null) {
 760                     // No nexthop selected yet, elect one now
 761                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 762                             null /*networkId*/, !isExternalSubnetVpn);
 763                 } else if (!isExternalSubnetVpn) {
 764                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 765                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 766                 }
 767             }
 768             subOpEntry = subOpBuilder.build();
 769             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 770                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 771             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 772                             + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 773                     LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 774                     subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 775                     subOpEntry.getLastAdvState());
 776         } catch (RuntimeException e) { //TODO: lockSubnet() throws this exception. Rectify lockSubnet()
 777             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}",
 778                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 779         } catch (TransactionCommitFailedException ex) {
 780             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to update subnetRoute for subnet {} on dpn {}",
 781                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 782         } catch (InterruptedException | ExecutionException e) {
 783             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to read data store for subnet {} on dpn {}",
 784                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 785         }
 786         finally {
 787             vpnUtil.unlockSubnet(subnetId.getValue());
 788         }
 789     }
 790
 791     // TODO Clean up the exception handling
 792     @SuppressWarnings("checkstyle:IllegalCatch")
 793     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, Uint64 dpnId) {
 794         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 795         //TODO(vivek): Change this to use more granularized lock at subnetId level
 796         try {
 797             vpnUtil.lockSubnet(subnetId.getValue());
 798             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 799                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 800                             new SubnetOpDataEntryKey(subnetId)).build();
 801             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 802                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 803             if (!optionalSubs.isPresent()) {
 804                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 805                         + " is not available", LOGGING_PREFIX, subnetId.getValue());
 806                 return;
 807             }
 808             LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 809                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 810                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 811                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 812                     optionalSubs.get().getLastAdvState());
 813             SubnetOpDataEntry subOpEntry = null;
 814             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 815             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 816             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 817                 electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 818                 subOpEntry = subOpBuilder.build();
 819                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 820                         subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 821                 LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 822                                 + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 823                         dpnId.toString(), optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 824                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 825                         optionalSubs.get().getLastAdvState());
 826             }
 827         } catch (RuntimeException e) {
 828             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 829                     + " dpnId {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 830         } catch (InterruptedException | ExecutionException e) {
 831             LOG.error("{} Failed to read data store for subnet {} dpn {}", LOGGING_PREFIX, subnetId, dpnId);
 832         } catch (TransactionCommitFailedException e) {
 833             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 834                     + " on dpn {} failed", LOGGING_PREFIX, subnetId.getValue(), dpnId, e);
 835         } finally {
 836             vpnUtil.unlockSubnet(subnetId.getValue());
 837         }
 838     }
 839
 840     @SuppressWarnings("checkstyle:IllegalCatch")
 841     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 842         try {
 843             //BGP manager will handle withdraw and advertise internally if prefix
 844             //already exist
 845             Uint32 label = Uint32.ZERO;
 846             Uint32 l3vni = Uint32.ZERO;
 847
 848             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 849             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 850                 l3vni = subOpBuilder.getL3vni();
 851             } else {
 852                 label = subOpBuilder.getLabel();
 853                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
 854                     LOG.error("publishSubnetRouteToBgp: Label not found for rd {}, subnetIp {}",
 855                             subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr());
 856                     return;
 857                 }
 858             }
 859             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 860                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 861                     Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 862             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 863         } catch (Exception e) {
 864             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 865                     + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 866                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 867         }
 868     }
 869
 870     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 871         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 872                 subOpBuilder.getNhDpnId());
 873         if (nhTepIp != null) {
 874             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 875         } else {
 876             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 877                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 878                     subOpBuilder.getNhDpnId().toString());
 879             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 880         }
 881     }
 882
 883     // TODO Clean up the exception handling
 884     @SuppressWarnings("checkstyle:IllegalCatch")
 885     private boolean addSubnetRouteToFib(String rd, String subnetIp, Uint64 nhDpnId, String nextHopIp,
 886                                         String vpnName, Long elanTag, Uint32 label, Uint32 l3vni,
 887                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 888
 889         Preconditions.checkNotNull(rd,
 890                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 891         Preconditions.checkNotNull(subnetIp,
 892                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 893         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 894         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 895         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 896         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 897         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 898         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 899                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 900         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 901                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni.longValue())
 902                 .setLabel(label.longValue()).setElanTag(elanTag)
 903                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 904         if (!isBgpVpn) {
 905             vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 906             return true;
 907         }
 908         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 909         vpnUtil.syncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil
 910                 .getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(vpnName), subnetIp), VpnUtil
 911                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, Prefixes.PrefixCue.SubnetRoute));
 912         vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 913         try {
 914             // BGP manager will handle withdraw and advertise internally if prefix
 915             // already exist
 916             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 917                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 918         } catch (Exception e) {
 919             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 920                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 921             return false;
 922         }
 923         return true;
 924     }
 925
 926     private Uint32 getLabel(String rd, String subnetIp) {
 927         Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME, VpnUtil.getNextHopLabelKey(rd, subnetIp));
 928         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 929                 subnetIp);
 930         return label;
 931     }
 932
 933     // TODO Clean up the exception handling
 934     @SuppressWarnings("checkstyle:IllegalCatch")
 935     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 936         Preconditions.checkNotNull(rd,
 937                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 938         Preconditions.checkNotNull(subnetIp,
 939                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 940         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 941         if (isBgpVpn) {
 942             try {
 943                 bgpManager.withdrawPrefix(rd, subnetIp);
 944             } catch (Exception e) {
 945                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}",
 946                         LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 947                 return false;
 948             }
 949         }
 950         return true;
 951     }
 952
 953     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 954         fibManager.removeFibEntry(rd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 955         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 956         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 957             String importingRd = vpnInstance.getVrfId();
 958             fibManager.removeFibEntry(importingRd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 959             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 960                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 961         }
 962         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 963                 + " vpnName {}", prefix, rd, vpnName);
 964     }
 965
 966     // TODO Clean up the exception handling
 967     @SuppressWarnings("checkstyle:IllegalCatch")
 968     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, @Nullable Uint64 oldDpnId,
 969                                            Uuid subnetId, Uuid networkId, boolean isBgpVpn) {
 970         boolean isRouteAdvertised = false;
 971         String rd = subOpBuilder.getVrfId();
 972         String subnetIp = subOpBuilder.getSubnetCidr();
 973         String vpnName = subOpBuilder.getVpnName();
 974         long elanTag = subOpBuilder.getElanTag().toJava();
 975         boolean isAlternateDpnSelected = false;
 976         Uint32 l3vni = Uint32.ZERO;
 977         Uint32 label = Uint32.ZERO;
 978         String networkName = networkId != null ? networkId.getValue() : null;
 979
 980         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} TaskState {}"
 981                 + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 982                 subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 983         if (!isBgpVpn) {
 984             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 985             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 986             // blended more better into this design.
 987             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 988                 l3vni = subOpBuilder.getL3vni();
 989             } else {
 990                 label = getLabel(rd, subnetIp);
 991                 subOpBuilder.setLabel(label);
 992             }
 993             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 994                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 995             if (isRouteAdvertised) {
 996                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 997             } else {
 998                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 999                     + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
1000                     vpnName, rd);
1001                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1002             }
1003             return;
1004         }
1005
1006         String nhTepIp = null;
1007         Uint64 nhDpnId = null;
1008         Map<SubnetToDpnKey, SubnetToDpn> toDpnKeySubnetToDpnMap = subOpBuilder.getSubnetToDpn();
1009         if (toDpnKeySubnetToDpnMap != null) {
1010             for (SubnetToDpn subnetToDpn : toDpnKeySubnetToDpnMap.values()) {
1011                 if (subnetToDpn.getDpnId().equals(oldDpnId)) {
1012                     // Is this same is as input dpnId, then ignore it
1013                     continue;
1014                 }
1015                 nhDpnId = subnetToDpn.getDpnId();
1016                 if (vpnNodeListener.isConnectedNode(nhDpnId)) {
1017                     // selected dpnId is connected to ODL
1018                     // but does it have a TEP configured at all?
1019                     try {
1020                         nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
1021                         if (nhTepIp != null) {
1022                             isAlternateDpnSelected = true;
1023                             break;
1024                         }
1025                     } catch (Exception e) {
1026                         LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
1027                                 + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
1028                     }
1029                 }
1030             }
1031         }
1032         if (!isAlternateDpnSelected) {
1033             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
1034             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
1035                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
1036                         + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName,
1037                         rd);
1038                 // Withdraw route from BGP for this subnet
1039                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
1040                 //subOpBuilder.setNhDpnId(Uint64.valueOf(null));
1041                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1042                 if (routeWithdrawn) {
1043                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
1044                 } else {
1045                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
1046                         + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
1047                         subnetIp, vpnName, rd);
1048                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
1049                 }
1050             }
1051         } else {
1052             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
1053             subOpBuilder.setNhDpnId(nhDpnId);
1054             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
1055                 l3vni = subOpBuilder.getL3vni();
1056             } else {
1057                 label = getLabel(rd, subnetIp);
1058                 subOpBuilder.setLabel(label);
1059             }
1060             //update the VRF entry for the subnetroute.
1061             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
1062                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
1063             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1064             if (isRouteAdvertised) {
1065                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1066             } else {
1067                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1068                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1069                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1070             }
1071         }
1072     }
1073
1074     private static boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1075         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1076                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1077     }
1078
1079     private static @NonNull List<SubnetToDpn> concat(@Nullable List<SubnetToDpn> list, @NonNull SubnetToDpn entry) {
1080         final List<SubnetToDpn> ret = new ArrayList<>();
1081         if (list != null) {
1082             ret.addAll(list);
1083         }
1084         ret.add(entry);
1085         return ret;
1086     }
1087 }
1088