vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.util.ArrayList;
  13 import java.util.Arrays;
  14 import java.util.Collections;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import java.util.Objects;
  19 import javax.inject.Inject;
  20 import javax.inject.Singleton;
  21 import org.eclipse.jdt.annotation.NonNull;
  22 import org.eclipse.jdt.annotation.Nullable;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  26 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  28 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  29 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  30 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  31 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  32 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  33 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  34 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  35 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  37 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  57 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  58 import org.opendaylight.yangtools.yang.common.Uint32;
  59 import org.opendaylight.yangtools.yang.common.Uint64;
  60 import org.slf4j.Logger;
  61 import org.slf4j.LoggerFactory;
  62
  63 @Singleton
  64 public class VpnSubnetRouteHandler {
  65     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  66     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  67     private final DataBroker dataBroker;
  68     private final SubnetOpDpnManager subOpDpnManager;
  69     private final IBgpManager bgpManager;
  70     private final VpnOpDataSyncer vpnOpDataSyncer;
  71     private final VpnNodeListener vpnNodeListener;
  72     private final IFibManager fibManager;
  73     private final VpnUtil vpnUtil;
  74
  75     @Inject
  76     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  77             final IBgpManager bgpManager, final VpnOpDataSyncer vpnOpDataSyncer, final VpnNodeListener vpnNodeListener,
  78             final IFibManager fibManager, VpnUtil vpnUtil) {
  79         this.dataBroker = dataBroker;
  80         this.subOpDpnManager = subnetOpDpnManager;
  81         this.bgpManager = bgpManager;
  82         this.vpnOpDataSyncer = vpnOpDataSyncer;
  83         this.vpnNodeListener = vpnNodeListener;
  84         this.fibManager = fibManager;
  85         this.vpnUtil = vpnUtil;
  86     }
  87
  88     // TODO Clean up the exception handling
  89     @SuppressWarnings("checkstyle:IllegalCatch")
  90     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  91         Uuid subnetId = subnetmap.getId();
  92         String subnetIp = subnetmap.getSubnetIp();
  93         Subnetmap subMap = null;
  94         SubnetOpDataEntry subOpEntry = null;
  95         SubnetOpDataEntryBuilder subOpBuilder = null;
  96         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
  97         Optional<SubnetOpDataEntry> optionalSubs = null;
  98
  99         Preconditions.checkNotNull(subnetId, LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 100         Preconditions.checkNotNull(subnetIp,
 101                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 102         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 103
 104         if (subnetmap.getVpnId() == null) {
 105             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 106                     subnetId);
 107             return;
 108         }
 109         String vpnName = subnetmap.getVpnId().getValue();
 110         Uint32 vpnId = waitAndGetVpnIdIfInvalid(vpnName);
 111         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 112             LOG.error(
 113                     "{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 114                             + "processing subnet {} with IP {}, bailing out now.",
 115                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 116             return;
 117         }
 118
 119         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 120         VpnInstanceOpDataEntry vpnInstanceOpData = waitAndGetVpnInstanceOpDataIfNull(vpnName, primaryRd);
 121         if (vpnInstanceOpData == null) {
 122             LOG.error(
 123                     "{} onSubnetAddedToVpn: VpnInstanceOpData not yet available for VpnName {} "
 124                             + "processing subnet {} with IP {}, bailing out now.",
 125                     LOGGING_PREFIX, vpnName, subnetId, subnetIp);
 126             return;
 127         }
 128         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 129                 subnetId.getValue(), subnetIp, vpnName);
 130         //TODO(vivek): Change this to use more granularized lock at subnetId level
 131         try {
 132             vpnUtil.lockSubnet(subnetId.getValue());
 133             // Please check if subnetId belongs to an External Network
 134             InstanceIdentifier<Subnetmap> subMapid =
 135                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 136                             new SubnetmapKey(subnetId)).build();
 137             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 138                     LogicalDatastoreType.CONFIGURATION, subMapid);
 139             if (!sm.isPresent()) {
 140                 LOG.error("{} onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet {} IP {}"
 141                         + " vpnName {}", LOGGING_PREFIX, subnetId, subnetIp, vpnName);
 142                 return;
 143             }
 144             subMap = sm.get();
 145
 146             if (isBgpVpn) {
 147                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 148                         .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 149                 Optional<Networks> optionalNets = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 150                         LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 151                 if (optionalNets.isPresent()) {
 152                     LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 153                                     + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 154                             subnetId.getValue(), subnetIp, subMap.getNetworkId().getValue(), vpnName);
 155                     return;
 156                 }
 157             }
 158             //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 159             subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 160                     new SubnetOpDataEntryKey(subnetId)).build();
 161             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 162                     subOpIdentifier);
 163             if (optionalSubs.isPresent()) {
 164                 LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 165                         + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 166                 return;
 167             }
 168             LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 169                     + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 170             subOpBuilder = new SubnetOpDataEntryBuilder().withKey(new SubnetOpDataEntryKey(subnetId));
 171             subOpBuilder.setSubnetId(subnetId);
 172             subOpBuilder.setSubnetCidr(subnetIp);
 173
 174             if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 175                 LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 176                         + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 177                 return;
 178             }
 179             subOpBuilder.setVrfId(primaryRd);
 180             subOpBuilder.setVpnName(vpnName);
 181             subOpBuilder.setSubnetToDpn(new ArrayList<>());
 182             subOpBuilder.setRouteAdvState(TaskState.Idle);
 183             subOpBuilder.setElanTag(elanTag);
 184             Long l3Vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni().toJava() : 0L;
 185             subOpBuilder.setL3vni(l3Vni);
 186             subOpEntry = subOpBuilder.build();
 187             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 188                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 189             LOG.info("{} onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}", LOGGING_PREFIX,
 190                     subnetId.getValue());
 191         } catch (TransactionCommitFailedException e) {
 192             LOG.error("{} Creation of SubnetOpDataEntry for subnet {} failed ", LOGGING_PREFIX,
 193                     subnetId.getValue(), e);
 194             // The second part of this method depends on subMap being non-null so fail fast here.
 195             return;
 196         } catch (RuntimeException e) { //TODO: Avoid this
 197             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 198                     subnetId.getValue(), subnetIp, vpnName, e);
 199             return;
 200         } catch (ReadFailedException e) {
 201             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 202                     subnetId, subnetIp, vpnName);
 203             return;
 204         } finally {
 205             vpnUtil.unlockSubnet(subnetId.getValue());
 206         }
 207         try {
 208             //In second critical section , Port-Op-Data will be updated.
 209             vpnUtil.lockSubnet(subnetId.getValue());
 210             Uint64 dpnId = null;
 211             SubnetToDpn subDpn = null;
 212             Map<Uint64, SubnetToDpn> subDpnMap = new HashMap<>();
 213
 214             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 215                     subOpIdentifier);
 216             subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get())
 217                     .withKey(new SubnetOpDataEntryKey(subnetId));
 218             List<Uuid> portList = subMap.getPortList();
 219             if (portList != null) {
 220                 for (Uuid port : portList) {
 221                     Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 222                     if (intfState != null) {
 223                         try {
 224                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 225                         } catch (Exception e) {
 226                             LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 227                                             + " subnetroute inclusion for this interface for subnet {} subnetIp {} "
 228                                     + "vpn {} failed with exception", LOGGING_PREFIX, port.getValue(),
 229                                     subnetId.getValue(), subnetIp, vpnName, e);
 230                             continue;
 231                         }
 232                         if (dpnId.equals(Uint64.ZERO)) {
 233                             LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 234                                             + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 235                                     subnetId.getValue(), subnetIp, vpnName);
 236                             continue;
 237                         }
 238                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 239                         if (intfState.getOperStatus() != OperStatus.Up) {
 240                             LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 241                                             + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(),
 242                                     subnetId.getValue(), subnetIp, vpnName);
 243                             continue;
 244                         }
 245                         subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 246                         if (intfState.getOperStatus() == OperStatus.Up) {
 247                             // port is UP
 248                             subDpnMap.put(dpnId, subDpn);
 249                         }
 250                     } else {
 251                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 252                     }
 253                 }
 254                 if (subDpnMap.size() > 0) {
 255                     subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 256                 }
 257             }
 258             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 259                     subMap.getNetworkId(), isBgpVpn);
 260             subOpEntry = subOpBuilder.build();
 261             SingleTransactionDataBroker.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 262                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 263             LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 264                             + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 265                     subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 266                     subOpEntry.getLastAdvState());
 267         } catch (RuntimeException e) {
 268             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 269                     subnetId.getValue(), subnetIp, vpnName, e);
 270         } catch (ReadFailedException e) {
 271             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 272                     subnetId, subnetIp, vpnName);
 273         } catch (TransactionCommitFailedException ex) {
 274             LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {} failed",
 275                     LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 276         } finally {
 277             vpnUtil.unlockSubnet(subnetId.getValue());
 278         }
 279     }
 280
 281     private Uint32 waitAndGetVpnIdIfInvalid(String vpnName) {
 282         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 283         if (VpnConstants.INVALID_ID.equals(vpnId)) {
 284             LOG.debug("VpnId is invalid, waiting to fetch again: vpnName={}, vpnId={}", vpnName, vpnId);
 285             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 286                     VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 287             vpnId = vpnUtil.getVpnId(vpnName);
 288         }
 289         return vpnId;
 290     }
 291
 292     private VpnInstanceOpDataEntry waitAndGetVpnInstanceOpDataIfNull(String vpnName, String primaryRd) {
 293         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 294         if (vpnInstanceOpData == null) {
 295             LOG.debug("vpnInstanceOpData is null, waiting to fetch again: vpnName={}", vpnName);
 296             vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnOpData, vpnName,
 297                     VpnConstants.PER_VPN_INSTANCE_OPDATA_MAX_WAIT_TIME_IN_MILLISECONDS);
 298             vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 299         }
 300         return vpnInstanceOpData;
 301     }
 302
 303     // TODO Clean up the exception handling
 304     @SuppressWarnings("checkstyle:IllegalCatch")
 305     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 306         Uuid subnetId = subnetmap.getId();
 307         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 308                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 309         //TODO(vivek): Change this to use more granularized lock at subnetId level
 310         try {
 311             vpnUtil.lockSubnet(subnetId.getValue());
 312             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 313                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 314                             new SubnetOpDataEntryKey(subnetId)).build();
 315             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 316                     LogicalDatastoreType.OPERATIONAL,
 317                     subOpIdentifier);
 318             if (!optionalSubs.isPresent()) {
 319                 LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 320                                 + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 321                         subnetId.getValue(), subnetmap.getVpnId());
 322                 return;
 323             }
 324             LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 325                             + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 326                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 327                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 328                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 329                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 330                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 331                  * going through the list of ports on the subnet
 332                  */
 333             InstanceIdentifier<Subnetmap> subMapid =
 334                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 335                             new SubnetmapKey(subnetId)).build();
 336             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 337                     LogicalDatastoreType.CONFIGURATION, subMapid);
 338             if (!sm.isPresent()) {
 339                 LOG.error("{} onSubnetDeletedFromVpn: Stale ports removal: Unable to retrieve subnetmap entry"
 340                                 + " for subnet {} subnetIp {} vpnName {}", LOGGING_PREFIX, subnetId.getValue(),
 341                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName());
 342             } else {
 343                 Subnetmap subMap = sm.get();
 344                 List<Uuid> portList = subMap.getPortList();
 345                 if (portList != null) {
 346                     for (Uuid port : portList) {
 347                         InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 348                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 349                                         new PortOpDataEntryKey(port.getValue())).build();
 350                         LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 351                                         + " from subnet {} subnetIp {} vpnName {} TaskState {}",
 352                                 LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 353                                 optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 354                                 optionalSubs.get().getRouteAdvState());
 355                         SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 356                                 portOpIdentifier, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 357                     }
 358                 }
 359             }
 360
 361             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 362             String rd = subOpBuilder.getVrfId();
 363             String subnetIp = subOpBuilder.getSubnetCidr();
 364             String vpnName = subOpBuilder.getVpnName();
 365             //Withdraw the routes for all the interfaces on this subnet
 366             //Remove subnet route entry from FIB
 367             deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 368             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 369                     VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 370             LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 371                             + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(),
 372                     subnetIp, vpnName, rd);
 373         } catch (RuntimeException e) { //TODO: Avoid this
 374             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {}",
 375                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 376         } catch (TransactionCommitFailedException ex) {
 377             LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 378                             + " vpnId {} failed", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 379                     subnetmap.getVpnId(), ex);
 380         } catch (ReadFailedException e) {
 381             LOG.error("{} onSubnetDeletedFromVpn: Failed to read data store for subnet {} ip {} vpn {}",
 382                     LOGGING_PREFIX, subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 383         } finally {
 384             vpnUtil.unlockSubnet(subnetId.getValue());
 385         }
 386     }
 387
 388     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 389         Uuid subnetId = subnetmap.getId();
 390         String vpnName = subnetmap.getVpnId().getValue();
 391         String subnetIp = subnetmap.getSubnetIp();
 392
 393         Preconditions.checkNotNull(subnetId,
 394                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 395         Preconditions.checkNotNull(subnetIp,
 396                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 397         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 398         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 399         try {
 400             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 401                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 402                             new SubnetOpDataEntryKey(subnetId)).build();
 403             Optional<SubnetOpDataEntry> optionalSubs =
 404                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 405                             subOpIdentifier);
 406             if (optionalSubs.isPresent()) {
 407                 onSubnetDeletedFromVpn(subnetmap, true);
 408             } else {
 409                 onSubnetAddedToVpn(subnetmap, true, elanTag);
 410             }
 411             LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 412                     subnetId.getValue(), subnetIp, vpnName);
 413         } catch (ReadFailedException e) {
 414             LOG.error("onSubnetUpdatedInVpn: Failed to read data store for subnet{} ip {} elanTag {} vpn {}",subnetId,
 415                     subnetIp, elanTag, vpnName);
 416         }
 417     }
 418
 419     // TODO Clean up the exception handling
 420     @SuppressWarnings("checkstyle:IllegalCatch")
 421     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 422         Uuid subnetId = subnetmap.getId();
 423         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 424                 subnetId.getValue());
 425         //TODO(vivek): Change this to use more granularized lock at subnetId level
 426         try {
 427             vpnUtil.lockSubnet(subnetId.getValue());
 428             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 429                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 430                             new SubnetOpDataEntryKey(subnetId)).build();
 431
 432             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 433                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 434             if (!optionalSubs.isPresent()) {
 435                 LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 436                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 437                 return;
 438             }
 439             String vpnName = optionalSubs.get().getVpnName();
 440             String subnetIp = optionalSubs.get().getSubnetCidr();
 441             String rd = optionalSubs.get().getVrfId();
 442             String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 443             LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 444                             + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 445                     vpnName, rd, routeAdvState);
 446             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 447             Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 448             if (intfState == null) {
 449                 // Interface State not yet available
 450                 return;
 451             }
 452             final Uint64 dpnId;
 453             try {
 454                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 455             } catch (Exception e) {
 456                 LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 457                                 + " for this interface failed for subnet {} subnetIp {} vpn {} rd {}",
 458                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd, e);
 459                 return;
 460             }
 461             if (dpnId.equals(Uint64.ZERO)) {
 462                 LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 463                                 + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 464                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 465                 return;
 466             }
 467             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 468             if (intfState.getOperStatus() != OperStatus.Up) {
 469                 LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 470                                 + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 471                         portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 472                 return;
 473             }
 474             LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 475                             + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 476                     subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 477             SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 478             if (subDpn == null) {
 479                 LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 480                                 + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 481                         portId.getValue(), vpnName, rd, subnetIp);
 482                 return;
 483             }
 484             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 485             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 486             subOpBuilder.setSubnetToDpn(concat(subnetOpDataEntry.getSubnetToDpn(), subDpn));
 487             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 488                 if (subOpBuilder.getNhDpnId() == null) {
 489                     // No nexthop selected yet, elect one now
 490                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 491                             subnetmap.getNetworkId(), true);
 492                 } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 493                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 494                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 495                 }
 496             }
 497             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 498             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 499                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 500             LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 501                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 502                     portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 503                     subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 504         } catch (RuntimeException e) { //TODO: Avoid this
 505             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {}", LOGGING_PREFIX,
 506                     portId.getValue(), subnetId.getValue(), e);
 507         } catch (ReadFailedException e) {
 508             LOG.error("{} onPortAddedToSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 509                     portId, subnetId);
 510         } catch (TransactionCommitFailedException e) {
 511             LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied",
 512                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), e);
 513         } finally {
 514             vpnUtil.unlockSubnet(subnetId.getValue());
 515         }
 516     }
 517
 518     // TODO Clean up the exception handling
 519     @SuppressWarnings("checkstyle:IllegalCatch")
 520     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 521         Uuid subnetId = subnetmap.getId();
 522         //TODO(vivek): Change this to use more granularized lock at subnetId level
 523         try {
 524             vpnUtil.lockSubnet(subnetId.getValue());
 525             PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 526                     subnetmap.getId());
 527             if (portOpEntry == null) {
 528                 return;
 529             }
 530             Uint64 dpnId = portOpEntry.getDpnId();
 531             if (dpnId == null) {
 532                 LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 533                                 + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 534                         subnetId.getValue());
 535                 return;
 536             }
 537             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 538             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 539                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 540                             new SubnetOpDataEntryKey(subnetId)).build();
 541             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 542                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 543             if (!optionalSubs.isPresent()) {
 544                 LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 545                         + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 546                 return;
 547             }
 548             LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 549                             + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 550                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 551                     optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 552                     optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 553             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 554             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 555             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 556             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 557                 // select another NhDpnId
 558                 if (last) {
 559                     LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 560                                     + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 561                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 562                     // last port on this DPN, so we need to elect the new NHDpnId
 563                     electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 564                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 565                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 566                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 567                     LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 568                                     + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 569                             portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 570                             subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 571                 }
 572             }
 573         } catch (RuntimeException e) {
 574             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {}", LOGGING_PREFIX,
 575                     portId.getValue(), subnetId.getValue(), e);
 576         } catch (ReadFailedException e) {
 577             LOG.error("{} onPortRemovedFromSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 578                     portId, subnetId);
 579         } catch (TransactionCommitFailedException e) {
 580             LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed", LOGGING_PREFIX,
 581                     portId.getValue(), subnetId.getValue(), e);
 582         } finally {
 583             vpnUtil.unlockSubnet(subnetId.getValue());
 584         }
 585     }
 586
 587     // TODO Clean up the exception handling
 588     @SuppressWarnings("checkstyle:IllegalCatch")
 589     public void onInterfaceUp(Uint64 dpnId, String intfName, Uuid subnetId) {
 590         //TODO(vivek): Change this to use more granularized lock at subnetId level
 591         SubnetToDpn subDpn = null;
 592         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 593             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 594                     intfName, subnetId.getValue());
 595             return;
 596         }
 597         try {
 598             vpnUtil.lockSubnet(subnetId.getValue());
 599             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 600                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 601                             new SubnetOpDataEntryKey(subnetId)).build();
 602             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 603                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 604             if (!optionalSubs.isPresent()) {
 605                 LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 606                         + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 607                 return;
 608             }
 609             subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 610             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 611             if (subDpn == null) {
 612                 return;
 613             }
 614             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 615             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 616             LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 617                             + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 618                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 619                     subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 620             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 621                     subnetId.getValue());
 622             subOpBuilder.setSubnetToDpn(concat(subnetOpDataEntry.getSubnetToDpn(), subDpn));
 623             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 624                 if (subOpBuilder.getNhDpnId() == null) {
 625                     // No nexthop selected yet, elect one now
 626                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 627                             null /*networkId*/, !isExternalSubnetVpn);
 628                 } else if (!isExternalSubnetVpn) {
 629                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 630                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 631                 }
 632             }
 633             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 634             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 635                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 636             LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 637                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 638                     subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 639                     subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 640         } catch (RuntimeException e) {
 641             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {}",
 642                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 643         } catch (ReadFailedException e) {
 644             LOG.error("{} onInterfaceUp: Failed to read data store for interface {} dpn {} subnet {}", LOGGING_PREFIX,
 645                     intfName, dpnId, subnetId);
 646         } catch (TransactionCommitFailedException e) {
 647             LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed",
 648                     LOGGING_PREFIX, subnetId.getValue(), intfName, e);
 649         } finally {
 650             vpnUtil.unlockSubnet(subnetId.getValue());
 651         }
 652     }
 653
 654     // TODO Clean up the exception handling
 655     @SuppressWarnings("checkstyle:IllegalCatch")
 656     public void onInterfaceDown(final Uint64 dpnId, final String interfaceName, Uuid subnetId) {
 657         if (dpnId == null || Objects.equals(dpnId, Uint64.ZERO)) {
 658             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 659                     interfaceName, subnetId.getValue());
 660             return;
 661         }
 662         try {
 663             vpnUtil.lockSubnet(subnetId.getValue());
 664             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 665             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 666                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 667                             new SubnetOpDataEntryKey(subnetId)).build();
 668             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 669                     LogicalDatastoreType.OPERATIONAL,
 670                     subOpIdentifier);
 671             if (!optionalSubs.isPresent()) {
 672                 LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 673                         + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 674                 return;
 675             }
 676             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 677             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 678             LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 679                             + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 680                     subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 681                     subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 682                     interfaceName);
 683             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 684             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 685                 // select another NhDpnId
 686                 if (last) {
 687                     LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 688                                     + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 689                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 690                     // last port on this DPN, so we need to elect the new NHDpnId
 691                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 692                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 693                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 694                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 695                     LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 696                                     + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 697                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 698                             interfaceName);
 699                 }
 700             }
 701         } catch (RuntimeException e) { //TODO: Remove RuntimeException
 702             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {}",
 703                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 704         } catch (ReadFailedException e) {
 705             LOG.error("{} onInterfaceDown: Failed to read data store for interface {} dpn {} subnet {}",
 706                     LOGGING_PREFIX, interfaceName, dpnId, subnetId.getValue(), e);
 707         } catch (TransactionCommitFailedException ex) {
 708             LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {} failed",
 709                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 710         } finally {
 711             vpnUtil.unlockSubnet(subnetId.getValue());
 712         }
 713     }
 714
 715     // TODO Clean up the exception handling
 716     @SuppressWarnings("checkstyle:IllegalCatch")
 717     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, Uint64 dpnId) {
 718         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 719                 dpnId.toString());
 720         try {
 721             vpnUtil.lockSubnet(subnetId.getValue());
 722             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 723                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 724                             new SubnetOpDataEntryKey(subnetId)).build();
 725             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 726                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 727             if (!optionalSubs.isPresent()) {
 728                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 729                         LOGGING_PREFIX, subnetId.getValue());
 730                 return;
 731             }
 732             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 733                             + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 734                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 735                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 736                     optionalSubs.get().getLastAdvState(), dpnId.toString());
 737             SubnetOpDataEntry subOpEntry = optionalSubs.get();
 738             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 739             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 740             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 741                 if (subOpBuilder.getNhDpnId() == null) {
 742                     // No nexthop selected yet, elect one now
 743                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 744                             null /*networkId*/, !isExternalSubnetVpn);
 745                 } else if (!isExternalSubnetVpn) {
 746                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 747                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 748                 }
 749             }
 750             subOpEntry = subOpBuilder.build();
 751             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 752                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 753             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 754                             + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 755                     LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 756                     subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 757                     subOpEntry.getLastAdvState());
 758         } catch (RuntimeException e) { //TODO: lockSubnet() throws this exception. Rectify lockSubnet()
 759             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}",
 760                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 761         } catch (TransactionCommitFailedException ex) {
 762             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to update subnetRoute for subnet {} on dpn {}",
 763                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 764         } catch (ReadFailedException e) {
 765             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to read data store for subnet {} on dpn {}",
 766                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 767         }
 768         finally {
 769             vpnUtil.unlockSubnet(subnetId.getValue());
 770         }
 771     }
 772
 773     // TODO Clean up the exception handling
 774     @SuppressWarnings("checkstyle:IllegalCatch")
 775     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, Uint64 dpnId) {
 776         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 777         //TODO(vivek): Change this to use more granularized lock at subnetId level
 778         try {
 779             vpnUtil.lockSubnet(subnetId.getValue());
 780             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 781                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 782                             new SubnetOpDataEntryKey(subnetId)).build();
 783             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 784                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 785             if (!optionalSubs.isPresent()) {
 786                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 787                         + " is not available", LOGGING_PREFIX, subnetId.getValue());
 788                 return;
 789             }
 790             LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 791                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 792                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 793                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 794                     optionalSubs.get().getLastAdvState());
 795             SubnetOpDataEntry subOpEntry = null;
 796             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 797             Uint64 nhDpnId = subOpBuilder.getNhDpnId();
 798             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 799                 electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 800                 subOpEntry = subOpBuilder.build();
 801                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 802                         subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 803                 LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 804                                 + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 805                         dpnId.toString(), optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 806                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 807                         optionalSubs.get().getLastAdvState());
 808             }
 809         } catch (RuntimeException e) {
 810             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 811                     + " dpnId {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 812         } catch (ReadFailedException e) {
 813             LOG.error("{} Failed to read data store for subnet {} dpn {}", LOGGING_PREFIX, subnetId, dpnId);
 814         } catch (TransactionCommitFailedException e) {
 815             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 816                     + " on dpn {} failed", LOGGING_PREFIX, subnetId.getValue(), dpnId, e);
 817         } finally {
 818             vpnUtil.unlockSubnet(subnetId.getValue());
 819         }
 820     }
 821
 822     @SuppressWarnings("checkstyle:IllegalCatch")
 823     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 824         try {
 825             //BGP manager will handle withdraw and advertise internally if prefix
 826             //already exist
 827             Uint32 label = Uint32.ZERO;
 828             Uint32 l3vni = Uint32.ZERO;
 829
 830             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 831             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 832                 l3vni = subOpBuilder.getL3vni();
 833             } else {
 834                 label = subOpBuilder.getLabel();
 835             }
 836             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 837                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 838                     Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 839             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 840         } catch (Exception e) {
 841             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 842                     + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 843                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 844         }
 845     }
 846
 847     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 848         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 849                 subOpBuilder.getNhDpnId());
 850         if (nhTepIp != null) {
 851             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 852         } else {
 853             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 854                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 855                     subOpBuilder.getNhDpnId().toString());
 856             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 857         }
 858     }
 859
 860     // TODO Clean up the exception handling
 861     @SuppressWarnings("checkstyle:IllegalCatch")
 862     private boolean addSubnetRouteToFib(String rd, String subnetIp, Uint64 nhDpnId, String nextHopIp,
 863                                         String vpnName, Long elanTag, Uint32 label, Uint32 l3vni,
 864                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 865
 866         Preconditions.checkNotNull(rd,
 867                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 868         Preconditions.checkNotNull(subnetIp,
 869                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 870         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 871         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 872         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 873         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 874         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 875         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 876                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 877         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 878                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni.longValue())
 879                 .setLabel(label.longValue()).setElanTag(elanTag)
 880                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 881         if (!isBgpVpn) {
 882             vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 883             return true;
 884         }
 885         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 886         vpnUtil.syncWrite(LogicalDatastoreType.OPERATIONAL, VpnUtil
 887                 .getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(vpnName), subnetIp), VpnUtil
 888                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, Prefixes.PrefixCue.SubnetRoute));
 889         vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 890         try {
 891             // BGP manager will handle withdraw and advertise internally if prefix
 892             // already exist
 893             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 894                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/, null /*gatewayMacAddress*/);
 895         } catch (Exception e) {
 896             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 897                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 898             return false;
 899         }
 900         return true;
 901     }
 902
 903     private Uint32 getLabel(String rd, String subnetIp) {
 904         Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME, VpnUtil.getNextHopLabelKey(rd, subnetIp));
 905         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 906                 subnetIp);
 907         return label;
 908     }
 909
 910     // TODO Clean up the exception handling
 911     @SuppressWarnings("checkstyle:IllegalCatch")
 912     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 913         Preconditions.checkNotNull(rd,
 914                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 915         Preconditions.checkNotNull(subnetIp,
 916                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 917         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 918         if (isBgpVpn) {
 919             try {
 920                 bgpManager.withdrawPrefix(rd, subnetIp);
 921             } catch (Exception e) {
 922                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}",
 923                         LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 924                 return false;
 925             }
 926         }
 927         return true;
 928     }
 929
 930     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 931         fibManager.removeFibEntry(rd, prefix, null);
 932         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 933         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 934             String importingRd = vpnInstance.getVrfId();
 935             fibManager.removeFibEntry(importingRd, prefix, null);
 936             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 937                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 938         }
 939         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 940                 + " vpnName {}", prefix, rd, vpnName);
 941     }
 942
 943     // TODO Clean up the exception handling
 944     @SuppressWarnings("checkstyle:IllegalCatch")
 945     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, @Nullable Uint64 oldDpnId,
 946                                            Uuid subnetId, Uuid networkId, boolean isBgpVpn) {
 947         List<SubnetToDpn> subDpnList = null;
 948         boolean isRouteAdvertised = false;
 949         subDpnList = new ArrayList<>();
 950         subDpnList = subOpBuilder.getSubnetToDpn() != null ? new ArrayList<>(subOpBuilder.getSubnetToDpn())
 951                 : subDpnList;
 952         String rd = subOpBuilder.getVrfId();
 953         String subnetIp = subOpBuilder.getSubnetCidr();
 954         String vpnName = subOpBuilder.getVpnName();
 955         long elanTag = subOpBuilder.getElanTag().toJava();
 956         boolean isAlternateDpnSelected = false;
 957         Uint32 l3vni = Uint32.ZERO;
 958         Uint32 label = Uint32.ZERO;
 959         String networkName = networkId != null ? networkId.getValue() : null;
 960
 961         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} TaskState {}"
 962                 + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 963                 subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 964         if (!isBgpVpn) {
 965             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 966             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 967             // blended more better into this design.
 968             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 969                 l3vni = subOpBuilder.getL3vni();
 970             } else {
 971                 label = getLabel(rd, subnetIp);
 972                 subOpBuilder.setLabel(label);
 973             }
 974             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 975                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 976             if (isRouteAdvertised) {
 977                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 978             } else {
 979                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 980                     + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 981                     vpnName, rd);
 982                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 983             }
 984             return;
 985         }
 986
 987         String nhTepIp = null;
 988         Uint64 nhDpnId = null;
 989         for (SubnetToDpn subnetToDpn : subDpnList) {
 990             if (subnetToDpn.getDpnId().equals(oldDpnId)) {
 991                 // Is this same is as input dpnId, then ignore it
 992                 continue;
 993             }
 994             nhDpnId = subnetToDpn.getDpnId();
 995             if (vpnNodeListener.isConnectedNode(nhDpnId)) {
 996                 // selected dpnId is connected to ODL
 997                 // but does it have a TEP configured at all?
 998                 try {
 999                     nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
1000                     if (nhTepIp != null) {
1001                         isAlternateDpnSelected = true;
1002                         break;
1003                     }
1004                 } catch (Exception e) {
1005                     LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
1006                             + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
1007                 }
1008             }
1009         }
1010         if (!isAlternateDpnSelected) {
1011             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
1012             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
1013                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
1014                         + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName,
1015                         rd);
1016                 // Withdraw route from BGP for this subnet
1017                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
1018                 //subOpBuilder.setNhDpnId(Uint64.valueOf(null));
1019                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1020                 if (routeWithdrawn) {
1021                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
1022                 } else {
1023                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
1024                         + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
1025                         subnetIp, vpnName, rd);
1026                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
1027                 }
1028             }
1029         } else {
1030             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
1031             subOpBuilder.setNhDpnId(nhDpnId);
1032             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
1033                 l3vni = subOpBuilder.getL3vni();
1034             } else {
1035                 label = getLabel(rd, subnetIp);
1036                 subOpBuilder.setLabel(label);
1037             }
1038             //update the VRF entry for the subnetroute.
1039             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
1040                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
1041             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1042             if (isRouteAdvertised) {
1043                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1044             } else {
1045                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1046                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1047                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1048             }
1049         }
1050     }
1051
1052     private static boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1053         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1054                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1055     }
1056
1057     private static @NonNull List<SubnetToDpn> concat(@Nullable List<SubnetToDpn> list, @NonNull SubnetToDpn entry) {
1058         final List<SubnetToDpn> ret = new ArrayList<>();
1059         if (list != null) {
1060             ret.addAll(list);
1061         }
1062         ret.add(entry);
1063         return ret;
1064     }
1065 }
1066