vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Preconditions;
  11 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.Collections;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Objects;
  19 import java.util.Optional;
  20 import java.util.Set;
  21 import java.util.concurrent.ExecutionException;
  22 import java.util.stream.Collectors;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.NonNull;
  26 import org.eclipse.jdt.annotation.Nullable;
  27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  28 import org.opendaylight.mdsal.binding.api.DataBroker;
  29 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  30 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  31 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  32 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  33 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  34 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  35 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  36 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  37 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  38 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  39 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  41 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpnKey;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  59 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  60 import org.opendaylight.yangtools.yang.common.Uint32;
  61 import org.opendaylight.yangtools.yang.common.Uint64;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 @Singleton
  66 public class VpnSubnetRouteHandler {
  67     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  68     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  69     private static final String VPN_EVENT_SOURCE_SUBNET_ROUTE = "vpnSubnetRouteEvent";
  70     private final DataBroker dataBroker;
  71     private final SubnetOpDpnManager subOpDpnManager;
  72     private final IBgpManager bgpManager;
  73     private final VpnOpDataSyncer vpnOpDataSyncer;
  74     private final VpnNodeListener vpnNodeListener;
  75     private final IFibManager fibManager;
  76     private final VpnUtil vpnUtil;
  77
  78     @Inject
  79     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  80                                  final IBgpManager bgpManager, final VpnOpDataSyncer vpnOpDataSyncer,
  81                                  final VpnNodeListener vpnNodeListener,
  82                                  final IFibManager fibManager, VpnUtil vpnUtil) {
  83         this.dataBroker = dataBroker;
  84         this.subOpDpnManager = subnetOpDpnManager;
  85         this.bgpManager = bgpManager;
  86         this.vpnOpDataSyncer = vpnOpDataSyncer;
  87         this.vpnNodeListener = vpnNodeListener;
  88         this.fibManager = fibManager;
  89         this.vpnUtil = vpnUtil;
  90     }
  91
  92     // TODO Clean up the exception handling
  93     @SuppressWarnings("checkstyle:IllegalCatch")
  94     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  95         Uuid subnetId = subnetmap.getId();
  96         String subnetIp = subnetmap.getSubnetIp();
  97         SubnetOpDataEntry subOpEntry = null;
  98         SubnetOpDataEntryBuilder subOpBuilder = null;
  99         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
 100         Optional<SubnetOpDataEntry> optionalSubs = null;
 101         Uint32 label;
 102
 103         Preconditions.checkNotNull(subnetId,
 104                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 105         Preconditions.checkNotNull(subnetIp,
 106                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 107         Preconditions.checkNotNull(elanTag,
 108                 LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 109
 110         if (subnetmap.getVpnId() == null) {
 111             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 112                     subnetId);
 113             return;
 114         }
 115         String vpnName = subnetmap.getVpnId().getValue();
 116         Uint32 vpnId = waitAndGetVpnIdIfInvalid(vpnName);
 117         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 118             LOG.error(
 119                     "{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 120                             + "processing subnet {} with IP {}, bailing out now.",
 121                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 122             return;
 123         }
 124
 125         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 126         VpnInstanceOpDataEntry vpnInstanceOpData = waitAndGetVpnInstanceOpDataIfNull(vpnName, primaryRd);
 127         if (vpnInstanceOpData == null) {
 128             LOG.error(
 129                     "{} onSubnetAddedToVpn: VpnInstanceOpData not yet available for VpnName {} "
 130                             + "processing subnet {} with IP {}, bailing out now.",
 131                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 132             return;
 133         }
 134         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 135                 subnetId.getValue(), subnetIp, vpnName);
 136         //TODO(vivek): Change this to use more granularized lock at subnetId level
 137         try {
 138             vpnUtil.lockSubnet(subnetId.getValue());
 139             if (isBgpVpn) {
 140                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 141                         .child(Networks.class, new NetworksKey(subnetmap.getNetworkId())).build();
 142                 Optional<Networks> optionalNets = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 143                         LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 144                 if (optionalNets.isPresent()) {
 145                     LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 146                                     + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 147                             subnetId.getValue(), subnetIp, subnetmap.getNetworkId().getValue(), vpnName);
 148                     return;
 149                 }
 150             }
 151             //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 152             subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 153                     new SubnetOpDataEntryKey(subnetId)).build();
 154             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 155                     subOpIdentifier);
 156             if (optionalSubs.isPresent()) {
 157                 LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 158                         + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 159                 return;
 160             }
 161             LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 162                     + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 163             subOpBuilder = new SubnetOpDataEntryBuilder().withKey(new SubnetOpDataEntryKey(subnetId));
 164             subOpBuilder.setSubnetId(subnetId);
 165             subOpBuilder.setSubnetCidr(subnetIp);
 166
 167             if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 168                 LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 169                         + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 170                 return;
 171             }
 172             //Allocate MPLS label for subnet-route at the time SubnetOpDataEntry creation
 173             label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 174                     VpnUtil.getNextHopLabelKey(primaryRd, subnetIp));
 175             if (label == VpnConstants.INVALID_ID) {
 176                 LOG.error("onSubnetAddedToVpn: Unable to retrieve label for rd {}, subnetIp {}",
 177                         primaryRd, subnetIp);
 178                 return;
 179             }
 180             subOpBuilder.setVrfId(primaryRd);
 181             subOpBuilder.setVpnName(vpnName);
 182             subOpBuilder.setSubnetToDpn(new ArrayList<>());
 183             subOpBuilder.setRouteAdvState(TaskState.Idle);
 184             subOpBuilder.setElanTag(elanTag);
 185             subOpBuilder.setLabel(label);
 186             Long l3Vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni().toJava() : 0L;
 187             subOpBuilder.setL3vni(l3Vni);
 188             subOpEntry = subOpBuilder.build();
 189             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 190                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 191             LOG.info("{} onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}", LOGGING_PREFIX,
 192                     subnetId.getValue());
 193         } catch (TransactionCommitFailedException e) {
 194             LOG.error("{} Creation of SubnetOpDataEntry for subnet {} failed ", LOGGING_PREFIX,
 195                     subnetId.getValue(), e);
 196             // The second part of this method depends on subnetmap being non-null so fail fast here.
 197             return;
 198         } catch (RuntimeException e) { //TODO: Avoid this
 199             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 200                     subnetId.getValue(), subnetIp, vpnName, e);
 201             return;
 202         } catch (InterruptedException | ExecutionException e) {
 203             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 204                     subnetId, subnetIp, vpnName);
 205             return;
 206         } finally {
 207             vpnUtil.unlockSubnet(subnetId.getValue());
 208         }
 209         try {
 210             //In second critical section , Port-Op-Data will be updated.
 211             vpnUtil.lockSubnet(subnetId.getValue());
 212             Uint64 dpnId = null;
 213             SubnetToDpn subDpn = null;
 214             Map<Uint64, SubnetToDpn> subDpnMap = new HashMap<>();
 215
 216             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 217                     subOpIdentifier);
 218             subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get())
 219                     .withKey(new SubnetOpDataEntryKey(subnetId));
 220             List<Uuid> portList = subnetmap.getPortList();
 221             if (portList != null) {
 222                 for (Uuid port : portList) {
 223                     Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 224                     if (intfState != null) {
 225                         try {
 226                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 227                         } catch (Exception e) {
 228                             LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 229                                             + " subnetroute inclusion for this interface for subnet {} subnetIp {} "
 230                                             + "vpn {} failed with exception", LOGGING_PREFIX, port.getValue(),
 231                                     subnetId.getValue(), subnetIp, vpnName, e);
 232                             continue;
 233                         }
 234                         if (dpnId.equals(Uint64.ZERO)) {
 235                             LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 236                                             + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 237                                     subnetId.getValue(), subnetIp, vpnName);
 238                             continue;
 239                         }
 240                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 241                         if (intfState.getOperStatus() != OperStatus.Up) {
 242                             LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 243                                             + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(),
 244                                     subnetId.getValue(), subnetIp, vpnName);
 245                             continue;
 246                         }
 247                         subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 248                         if (intfState.getOperStatus() == OperStatus.Up) {
 249                             // port is UP
 250                             subDpnMap.put(dpnId, subDpn);
 251                         }
 252                     } else {
 253                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 254                     }
 255                 }
 256                 if (subDpnMap.size() > 0) {
 257                     subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 258                 }
 259             }
 260             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 261                     subnetmap.getNetworkId(), isBgpVpn, label);
 262             subOpEntry = subOpBuilder.build();
 263             SingleTransactionDataBroker.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 264                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 265             LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 266                             + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 267                     subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 268                     subOpEntry.getLastAdvState());
 269         } catch (RuntimeException e) {
 270             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 271                     subnetId.getValue(), subnetIp, vpnName, e);
 272         } catch (InterruptedException | ExecutionException e) {
 273             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 274                     subnetId, subnetIp, vpnName);
 275         } catch (TransactionCommitFailedException ex) {
 276             LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {} failed",
 277                     LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 278         } finally {
 279             vpnUtil.unlockSubnet(subnetId.getValue());
 280         }
 281     }
 282
 283     private Uint32 waitAndGetVpnIdIfInvalid(String vpnName) {
 284         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 285         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 286             LOG.debug("VpnId is invalid, waiting to fetch again: vpnName={}, vpnId={}", vpnName, vpnId);
 287             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 288                     VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 289             vpnId = vpnUtil.getVpnId(vpnName);
 290         }
 291         return vpnId;
 292     }
 293
 294     private VpnInstanceOpDataEntry waitAndGetVpnInstanceOpDataIfNull(String vpnName, String primaryRd) {
 295         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 296         if (vpnInstanceOpData == null) {
 297             LOG.debug("vpnInstanceOpData is null, waiting to fetch again: vpnName={}", vpnName);
 298             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnOpData, vpnName,
 299                     VpnConstants.PER_VPN_INSTANCE_OPDATA_MAX_WAIT_TIME_IN_MILLISECONDS);
 300             vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 301         }
 302         return vpnInstanceOpData;
 303     }
 304
 305     // TODO Clean up the exception handling
 306     @SuppressWarnings("checkstyle:IllegalCatch")
 307     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 308         Uuid subnetId = subnetmap.getId();
 309         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 310                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 311         //TODO(vivek): Change this to use more granularized lock at subnetId level
 312         try {
 313             vpnUtil.lockSubnet(subnetId.getValue());
 314             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 315                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 316                             new SubnetOpDataEntryKey(subnetId)).build();
 317             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 318                     LogicalDatastoreType.OPERATIONAL,
 319                     subOpIdentifier);
 320             if (!optionalSubs.isPresent()) {
 321                 LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 322                                 + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 323                         subnetId.getValue(), subnetmap.getVpnId());
 324                 return;
 325             }
 326             LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 327                             + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 328                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 329                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 330             /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 331              * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 332              * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 333              * going through the list of ports on the subnet
 334              */
 335             List<Uuid> portList = subnetmap.getPortList();
 336             if (portList != null) {
 337                 for (Uuid port : portList) {
 338                     InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 339                             InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 340                                     new PortOpDataEntryKey(port.getValue())).build();
 341                     LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 342                                     + " from subnet {} subnetIp {} vpnName {} TaskState {}",
 343                             LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 344                             optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 345                             optionalSubs.get().getRouteAdvState());
 346                     SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 347                             portOpIdentifier, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 348                 }
 349             }
 350
 351             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 352             String rd = subOpBuilder.getVrfId();
 353             String subnetIp = subOpBuilder.getSubnetCidr();
 354             String vpnName = subOpBuilder.getVpnName();
 355             //Withdraw the routes for all the interfaces on this subnet
 356             //Remove subnet route entry from FIB
 357             deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 358             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 359                     VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 360             int releasedLabel = vpnUtil.releaseId(VpnConstants.VPN_IDPOOL_NAME,
 361                     VpnUtil.getNextHopLabelKey(rd, subnetIp));
 362             if (releasedLabel == VpnConstants.INVALID_LABEL) {
 363                 LOG.error("onSubnetDeletedFromVpn: Unable to release label for key {}",
 364                         VpnUtil.getNextHopLabelKey(rd, subnetIp));
 365             }
 366             LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 367                             + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(),
 368                     subnetIp, vpnName, rd);
 369         } catch (RuntimeException e) { //TODO: Avoid this
 370             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {}",
 371                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 372         } catch (TransactionCommitFailedException ex) {
 373             LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 374                             + " vpnId {} failed", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 375                     subnetmap.getVpnId(), ex);
 376         } catch (InterruptedException | ExecutionException e) {
 377             LOG.error("{} onSubnetDeletedFromVpn: Failed to read data store for subnet {} ip {} vpn {}",
 378                     LOGGING_PREFIX, subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 379         } finally {
 380             vpnUtil.unlockSubnet(subnetId.getValue());
 381         }
 382     }
 383
 384     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 385         Uuid subnetId = subnetmap.getId();
 386         String vpnName = subnetmap.getVpnId().getValue();
 387         String subnetIp = subnetmap.getSubnetIp();
 388
 389         Preconditions.checkNotNull(subnetId,
 390                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 391         Preconditions.checkNotNull(subnetIp,
 392                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 393         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 394         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 395         try {
 396             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 397                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 398                             new SubnetOpDataEntryKey(subnetId)).build();
 399             Optional<SubnetOpDataEntry> optionalSubs =
 400                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 401                             subOpIdentifier);
 402             if (optionalSubs.isPresent()) {
 403                 onSubnetDeletedFromVpn(subnetmap, true);
 404             } else {
 405                 onSubnetAddedToVpn(subnetmap, true, elanTag);
 406             }
 407             LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 408                     subnetId.getValue(), subnetIp, vpnName);
 409         } catch (InterruptedException | ExecutionException e) {
 410             LOG.error("onSubnetUpdatedInVpn: Failed to read data store for subnet{} ip {} elanTag {} vpn {}",subnetId,
 411                     subnetIp, elanTag, vpnName);
 412         }
 413     }
 414
 415     // TODO Clean up the exception handling
 416     @SuppressWarnings("checkstyle:IllegalCatch")
 417     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 418         Uuid subnetId = subnetmap.getId();
 419         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 420                 subnetId.getValue());
 421         //TODO(vivek): Change this to use more granularized lock at subnetId level
 422         try {
 423             vpnUtil.lockSubnet(subnetId.getValue());
 424             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 425                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 426                             new SubnetOpDataEntryKey(subnetId)).build();
 427
 428             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 429                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 430             if (!optionalSubs.isPresent()) {
 431                 LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 432                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 433                 return;
 434             }
 435             String vpnName = optionalSubs.get().getVpnName();
 436             String subnetIp = optionalSubs.get().getSubnetCidr();
 437             String rd = optionalSubs.get().getVrfId();
 438             String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 439             Uint32 label = optionalSubs.get().getLabel();
 440             LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 441                             + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 442                     vpnName, rd, routeAdvState);
 443             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 444             Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 445             if (intfState == null) {
 446                 // Interface State not yet available
 447                 return;
 448             }
 449             final Uint64 dpnId;
 450             try {
 451                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 452             } catch (Exception e) {
 453                 LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 454                                 + " for this interface failed for subnet {} subnetIp {} vpn {} rd {}",
 455                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd, e);
 456                 return;
 457             }
 458             if (dpnId.equals(Uint64.ZERO)) {
 459                 LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 460                                 + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 461                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 462                 return;
 463             }
 464             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 465             if (intfState.getOperStatus() != OperStatus.Up) {
 466                 LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 467                                 + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 468                         portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 469                 return;
 470             }
 471             LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 472                             + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 473                     subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 474             SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 475             if (subDpn == null) {
 476                 LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 477                                 + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 478                         portId.getValue(), vpnName, rd, subnetIp);
 479                 return;
 480             }
 481             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 482             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 483             List<SubnetToDpn> subnetToDpnList = concat(new ArrayList<SubnetToDpn>(subnetOpDataEntry
 484                     .nonnullSubnetToDpn().values()), subDpn);
 485             subOpBuilder.setSubnetToDpn(getSubnetToDpnMap(subnetToDpnList));
 486             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 487                 if (subOpBuilder.getNhDpnId() == null) {
 488                     // No nexthop selected yet, elect one now
 489                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 490                             subnetmap.getNetworkId(), true, label);
 491                 } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 492                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 493                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 494                 }
 495             }
 496             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 497             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 498                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 499             LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 500                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 501                     portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 502                     subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 503         } catch (RuntimeException e) { //TODO: Avoid this
 504             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {}", LOGGING_PREFIX,
 505                     portId.getValue(), subnetId.getValue(), e);
 506         } catch (InterruptedException | ExecutionException e) {
 507             LOG.error("{} onPortAddedToSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 508                     portId, subnetId);
 509         } catch (TransactionCommitFailedException e) {
 510             LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied",
 511                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), e);
 512         } finally {
 513             vpnUtil.unlockSubnet(subnetId.getValue());
 514         }
 515     }
 516
 517     // TODO Clean up the exception handling
 518     @SuppressWarnings("checkstyle:IllegalCatch")
 519     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 520         Uuid subnetId = subnetmap.getId();
 521         //TODO(vivek): Change this to use more granularized lock at subnetId level
 522         try {
 523             vpnUtil.lockSubnet(subnetId.getValue());
 524             PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 525                     subnetmap.getId());
 526             if (portOpEntry == null) {
 527                 return;
 528             }
 529             Uint64 dpnId = portOpEntry.getDpnId();
 530             if (dpnId == null) {
 531                 LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 532                                 + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 533                         subnetId.getValue());
 534                 return;
 535             }
 536             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 537             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 538                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 539                             new SubnetOpDataEntryKey(subnetId)).build();
 540             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 541                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 542             if (!optionalSubs.isPresent()) {
 543                 LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 544                         + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 545                 return;
 546             }
 547             LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 548                             + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 549                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 550                     optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 551                     optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 552             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 553             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 554             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 555             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 556                 // select another NhDpnId
 557                 if (last) {
 558                     LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 559                                     + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 560                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 561                     // last port on this DPN, so we need to elect the new NHDpnId
 562                     electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 563                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()),
 564                             subOpBuilder.getLabel());
 565                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 566                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 567                     LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 568                                     + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 569                             portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 570                             subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 571                 }
 572             }
 573         } catch (RuntimeException e) {
 574             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {}", LOGGING_PREFIX,
 575                     portId.getValue(), subnetId.getValue(), e);
 576         } catch (InterruptedException | ExecutionException e) {
 577             LOG.error("{} onPortRemovedFromSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 578                     portId, subnetId);
 579         } catch (TransactionCommitFailedException e) {
 580             LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed", LOGGING_PREFIX,
 581                     portId.getValue(), subnetId.getValue(), e);
 582         } finally {
 583             vpnUtil.unlockSubnet(subnetId.getValue());
 584         }
 585     }
 586
 587     // TODO Clean up the exception handling
 588     @SuppressWarnings("checkstyle:IllegalCatch")
 589     public void onInterfaceUp(Uint64 dpnId, String intfName, Uuid subnetId) {
 590         //TODO(vivek): Change this to use more granularized lock at subnetId level
 591         SubnetToDpn subDpn = null;
 592         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 593             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 594                     intfName, subnetId.getValue());
 595             return;
 596         }
 597         try {
 598             vpnUtil.lockSubnet(subnetId.getValue());
 599             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 600                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 601                             new SubnetOpDataEntryKey(subnetId)).build();
 602             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 603                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 604             if (!optionalSubs.isPresent()) {
 605                 LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 606                         + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 607                 return;
 608             }
 609             subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 610             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 611             if (subDpn == null) {
 612                 return;
 613             }
 614             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 615             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 616             LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 617                             + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 618                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 619                     subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 620             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 621                     subnetId.getValue());
 622             List<SubnetToDpn> subnetToDpnList = concat(new ArrayList<SubnetToDpn>(subnetOpDataEntry
 623                     .nonnullSubnetToDpn().values()), subDpn);
 624             subOpBuilder.setSubnetToDpn(getSubnetToDpnMap(subnetToDpnList));
 625             Uint32 label = optionalSubs.get().getLabel();
 626             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 627                 if (subOpBuilder.getNhDpnId() == null) {
 628                     // No nexthop selected yet, elect one now
 629                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 630                             null /*networkId*/, !isExternalSubnetVpn, label);
 631                 } else if (!isExternalSubnetVpn) {
 632                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 633                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 634                 }
 635             }
 636             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 637             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 638                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 639             LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 640                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 641                     subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 642                     subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 643         } catch (RuntimeException e) {
 644             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {}",
 645                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 646         } catch (InterruptedException | ExecutionException e) {
 647             LOG.error("{} onInterfaceUp: Failed to read data store for interface {} dpn {} subnet {}", LOGGING_PREFIX,
 648                     intfName, dpnId, subnetId);
 649         } catch (TransactionCommitFailedException e) {
 650             LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed",
 651                     LOGGING_PREFIX, subnetId.getValue(), intfName, e);
 652         } finally {
 653             vpnUtil.unlockSubnet(subnetId.getValue());
 654         }
 655     }
 656
 657     private Map<SubnetToDpnKey, SubnetToDpn> getSubnetToDpnMap(List<SubnetToDpn> subnetToDpnList) {
 658         //convert to set to remove duplicates.
 659         Set<SubnetToDpn> subnetToDpnSet = subnetToDpnList.stream().collect(Collectors.toSet());
 660         Map<SubnetToDpnKey, SubnetToDpn> subnetToDpnMap = new HashMap<>();
 661         for (SubnetToDpn subnetToDpn : subnetToDpnSet) {
 662             subnetToDpnMap.put(new SubnetToDpnKey(subnetToDpn.key()), subnetToDpn);
 663         }
 664         return subnetToDpnMap;
 665     }
 666
 667     // TODO Clean up the exception handling
 668     @SuppressWarnings("checkstyle:IllegalCatch")
 669     public void onInterfaceDown(final Uint64 dpnId, final String interfaceName, Uuid subnetId) {
 670         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 671             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 672                     interfaceName, subnetId.getValue());
 673             return;
 674         }
 675         try {
 676             vpnUtil.lockSubnet(subnetId.getValue());
 677             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 678             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 679                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 680                             new SubnetOpDataEntryKey(subnetId)).build();
 681             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 682                     LogicalDatastoreType.OPERATIONAL,
 683                     subOpIdentifier);
 684             if (!optionalSubs.isPresent()) {
 685                 LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 686                         + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 687                 return;
 688             }
 689             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 690             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 691             LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 692                             + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 693                     subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 694                     subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 695                     interfaceName);
 696             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 697             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 698                 // select another NhDpnId
 699                 if (last) {
 700                     LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 701                                     + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 702                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 703                     // last port on this DPN, so we need to elect the new NHDpnId
 704                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 705                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()),
 706                             subOpBuilder.getLabel());
 707                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 708                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 709                     LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 710                                     + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 711                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 712                             interfaceName);
 713                 }
 714             }
 715         } catch (RuntimeException e) { //TODO: Remove RuntimeException
 716             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {}",
 717                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 718         } catch (InterruptedException | ExecutionException e) {
 719             LOG.error("{} onInterfaceDown: Failed to read data store for interface {} dpn {} subnet {}",
 720                     LOGGING_PREFIX, interfaceName, dpnId, subnetId.getValue(), e);
 721         } catch (TransactionCommitFailedException ex) {
 722             LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {} failed",
 723                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 724         } finally {
 725             vpnUtil.unlockSubnet(subnetId.getValue());
 726         }
 727     }
 728
 729     // TODO Clean up the exception handling
 730     @SuppressWarnings("checkstyle:IllegalCatch")
 731     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, Uint64 dpnId) {
 732         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 733                 dpnId.toString());
 734         try {
 735             vpnUtil.lockSubnet(subnetId.getValue());
 736             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 737                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 738                             new SubnetOpDataEntryKey(subnetId)).build();
 739             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 740                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 741             if (!optionalSubs.isPresent()) {
 742                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 743                         LOGGING_PREFIX, subnetId.getValue());
 744                 return;
 745             }
 746             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 747                             + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 748                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 749                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 750                     optionalSubs.get().getLastAdvState(), dpnId.toString());
 751             SubnetOpDataEntry subOpEntry = optionalSubs.get();
 752             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 753             Uint32 label = optionalSubs.get().getLabel();
 754             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 755             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 756                 if (subOpBuilder.getNhDpnId() == null) {
 757                     // No nexthop selected yet, elect one now
 758                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 759                             null /*networkId*/, !isExternalSubnetVpn, label);
 760                 } else if (!isExternalSubnetVpn) {
 761                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 762                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 763                 }
 764             }
 765             subOpEntry = subOpBuilder.build();
 766             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 767                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 768             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 769                             + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 770                     LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 771                     subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 772                     subOpEntry.getLastAdvState());
 773         } catch (RuntimeException e) { //TODO: lockSubnet() throws this exception. Rectify lockSubnet()
 774             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}",
 775                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 776         } catch (TransactionCommitFailedException ex) {
 777             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to update subnetRoute for subnet {} on dpn {}",
 778                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 779         } catch (InterruptedException | ExecutionException e) {
 780             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to read data store for subnet {} on dpn {}",
 781                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 782         }
 783         finally {
 784             vpnUtil.unlockSubnet(subnetId.getValue());
 785         }
 786     }
 787
 788     // TODO Clean up the exception handling
 789     @SuppressWarnings("checkstyle:IllegalCatch")
 790     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, Uint64 dpnId) {
 791         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 792         //TODO(vivek): Change this to use more granularized lock at subnetId level
 793         try {
 794             vpnUtil.lockSubnet(subnetId.getValue());
 795             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 796                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 797                             new SubnetOpDataEntryKey(subnetId)).build();
 798             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 799                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 800             if (!optionalSubs.isPresent()) {
 801                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 802                         + " is not available", LOGGING_PREFIX, subnetId.getValue());
 803                 return;
 804             }
 805             LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 806                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 807                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 808                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 809                     optionalSubs.get().getLastAdvState());
 810             SubnetOpDataEntry subOpEntry = null;
 811             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 812             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 813             Uint32 label = optionalSubs.get().getLabel();
 814             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 815                 electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true,
 816                         label);
 817                 subOpEntry = subOpBuilder.build();
 818                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 819                         subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 820                 LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 821                                 + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 822                         dpnId.toString(), optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 823                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 824                         optionalSubs.get().getLastAdvState());
 825             }
 826         } catch (RuntimeException e) {
 827             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 828                     + " dpnId {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 829         } catch (InterruptedException | ExecutionException e) {
 830             LOG.error("{} Failed to read data store for subnet {} dpn {}", LOGGING_PREFIX, subnetId, dpnId);
 831         } catch (TransactionCommitFailedException e) {
 832             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 833                     + " on dpn {} failed", LOGGING_PREFIX, subnetId.getValue(), dpnId, e);
 834         } finally {
 835             vpnUtil.unlockSubnet(subnetId.getValue());
 836         }
 837     }
 838
 839     @SuppressWarnings("checkstyle:IllegalCatch")
 840     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 841         try {
 842             //BGP manager will handle withdraw and advertise internally if prefix
 843             //already exist
 844             Uint32 label = Uint32.ZERO;
 845             Uint32 l3vni = Uint32.ZERO;
 846
 847             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 848             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 849                 l3vni = subOpBuilder.getL3vni();
 850             } else {
 851                 label = subOpBuilder.getLabel();
 852                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
 853                     LOG.error("publishSubnetRouteToBgp: Label not found for rd {}, subnetIp {}",
 854                             subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr());
 855                     return;
 856                 }
 857             }
 858             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 859                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 860                     Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 861             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 862         } catch (Exception e) {
 863             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 864                             + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 865                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 866         }
 867     }
 868
 869     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 870         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 871                 subOpBuilder.getNhDpnId());
 872         if (nhTepIp != null) {
 873             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 874         } else {
 875             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 876                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 877                     subOpBuilder.getNhDpnId().toString());
 878             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/,
 879                     true, subOpBuilder.getLabel());
 880         }
 881     }
 882
 883     // TODO Clean up the exception handling
 884     @SuppressWarnings("checkstyle:IllegalCatch")
 885     private boolean addSubnetRouteToFib(String rd, String subnetIp, Uint64 nhDpnId, String nextHopIp,
 886                                         String vpnName, Long elanTag, Uint32 label, Uint32 l3vni,
 887                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 888
 889         Preconditions.checkNotNull(rd,
 890                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 891         Preconditions.checkNotNull(subnetIp,
 892                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 893         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 894         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 895         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 896         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 897         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 898         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 899                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 900         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 901                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni.longValue())
 902                 .setLabel(label.longValue()).setElanTag(elanTag)
 903                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 904         if (!isBgpVpn) {
 905             vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 906             return true;
 907         }
 908         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 909         vpnUtil.syncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil
 910                 .getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(vpnName), subnetIp), VpnUtil
 911                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, Prefixes.PrefixCue.SubnetRoute));
 912         vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 913         try {
 914             // BGP manager will handle withdraw and advertise internally if prefix
 915             // already exist
 916             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 917                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 918         } catch (Exception e) {
 919             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 920                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 921             return false;
 922         }
 923         return true;
 924     }
 925
 926     @SuppressFBWarnings
 927     private Uint32 getLabel(String rd, String subnetIp) {
 928         Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME, VpnUtil.getNextHopLabelKey(rd, subnetIp));
 929         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 930                 subnetIp);
 931         return label;
 932     }
 933
 934     // TODO Clean up the exception handling
 935     @SuppressWarnings("checkstyle:IllegalCatch")
 936     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 937         Preconditions.checkNotNull(rd,
 938                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 939         Preconditions.checkNotNull(subnetIp,
 940                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 941         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 942         if (isBgpVpn) {
 943             try {
 944                 bgpManager.withdrawPrefix(rd, subnetIp);
 945             } catch (Exception e) {
 946                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}",
 947                         LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 948                 return false;
 949             }
 950         }
 951         return true;
 952     }
 953
 954     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 955         fibManager.removeFibEntry(rd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 956         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 957         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 958             String importingRd = vpnInstance.getVrfId();
 959             fibManager.removeFibEntry(importingRd, prefix, VPN_EVENT_SOURCE_SUBNET_ROUTE, null);
 960             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 961                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 962         }
 963         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 964                 + " vpnName {}", prefix, rd, vpnName);
 965     }
 966
 967     // TODO Clean up the exception handling
 968     @SuppressWarnings("checkstyle:IllegalCatch")
 969     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, @Nullable Uint64 oldDpnId,
 970                                            Uuid subnetId, Uuid networkId, boolean isBgpVpn, Uint32 label) {
 971         boolean isRouteAdvertised = false;
 972         String rd = subOpBuilder.getVrfId();
 973         String subnetIp = subOpBuilder.getSubnetCidr();
 974         String vpnName = subOpBuilder.getVpnName();
 975         long elanTag = subOpBuilder.getElanTag().toJava();
 976         boolean isAlternateDpnSelected = false;
 977         Uint32 l3vni = Uint32.ZERO;
 978         String networkName = networkId != null ? networkId.getValue() : null;
 979
 980         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} label {} TaskState {}"
 981                         + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 982                 subOpBuilder.getVrfId(), label, subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 983         if (!isBgpVpn) {
 984             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 985             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 986             // blended more better into this design.
 987             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 988                 l3vni = subOpBuilder.getL3vni();
 989             } else {
 990                 subOpBuilder.setLabel(label);
 991             }
 992             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 993                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 994             if (isRouteAdvertised) {
 995                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 996             } else {
 997                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 998                                 + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 999                         vpnName, rd);
1000                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1001             }
1002             return;
1003         }
1004
1005         String nhTepIp = null;
1006         Uint64 nhDpnId = null;
1007         Map<SubnetToDpnKey, SubnetToDpn> toDpnKeySubnetToDpnMap = subOpBuilder.getSubnetToDpn();
1008         if (toDpnKeySubnetToDpnMap != null) {
1009             for (SubnetToDpn subnetToDpn : toDpnKeySubnetToDpnMap.values()) {
1010                 if (subnetToDpn.getDpnId().equals(oldDpnId)) {
1011                     // Is this same is as input dpnId, then ignore it
1012                     continue;
1013                 }
1014                 nhDpnId = subnetToDpn.getDpnId();
1015                 if (vpnNodeListener.isConnectedNode(nhDpnId)) {
1016                     // selected dpnId is connected to ODL
1017                     // but does it have a TEP configured at all?
1018                     try {
1019                         nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
1020                         if (nhTepIp != null) {
1021                             isAlternateDpnSelected = true;
1022                             break;
1023                         }
1024                     } catch (Exception e) {
1025                         LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
1026                                 + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
1027                     }
1028                 }
1029             }
1030         }
1031         if (!isAlternateDpnSelected) {
1032             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
1033             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
1034                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
1035                                 + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
1036                         vpnName, rd);
1037                 // Withdraw route from BGP for this subnet
1038                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
1039                 //subOpBuilder.setNhDpnId(Uint64.valueOf(null));
1040                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1041                 if (routeWithdrawn) {
1042                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
1043                 } else {
1044                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
1045                                     + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
1046                             subnetIp, vpnName, rd);
1047                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
1048                 }
1049             }
1050         } else {
1051             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
1052             subOpBuilder.setNhDpnId(nhDpnId);
1053             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
1054                 l3vni = subOpBuilder.getL3vni();
1055             } else {
1056                 subOpBuilder.setLabel(label);
1057             }
1058             //update the VRF entry for the subnetroute.
1059             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
1060                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
1061             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1062             if (isRouteAdvertised) {
1063                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1064             } else {
1065                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1066                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1067                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1068             }
1069         }
1070     }
1071
1072     private static boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1073         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1074                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1075     }
1076
1077     private static @NonNull List<SubnetToDpn> concat(@Nullable List<SubnetToDpn> list, @NonNull SubnetToDpn entry) {
1078         final List<SubnetToDpn> ret = new ArrayList<>();
1079         if (list != null) {
1080             ret.addAll(list);
1081         }
1082         ret.add(entry);
1083         return ret;
1084     }
1085 }
1086